class RPCProxy(object):
def __init__(self, uri, method=None, namespaces=None, sid=''):
self.__serviceURL = uri
self.__serviceName = method
self.namespaces = isinstance(namespaces, ClassLoader) and namespaces or \
ClassLoader(namespaces or [])
self._seril = Serializer(self.namespaces)
self.sid = sid
self.logger = None
self.start_call_listener = []
self.end_call_listener = []
def __call__(self, *args, **kw):
args = self._seril.serialize((args, kw))
post_data = {"method": self.__serviceName,
'params': args,
'id':'httprpc',
'sid':self.sid}
#@todo xx
for l in self.start_call_listener:
l(name=self.__serviceName, args=args, kw_args=kw)
rpc_response = self.post_request(self.__serviceURL, post_data)
try:
respdata = rpc_response.read()
ret, e = self._seril.deserialize(respdata)
except Exception, e:
raise RPCException("Failed to deserialize response data:%s, exception:%s" %
(respdata, e))
finally:
def testSimpleSerialization(self):
m = self.createTestMessage()
serialized = StringIO(Serializer.serialize(m))
deserialized = Serializer.deserialize(serialized, TestMessage1)
self.assertDictEqual(m.__dict__, deserialized.__dict__)
def __init__(self):
tag = 5007
fields = [{
'name': 'blockLength',
'fmt': 'H'
}, {
'name': 'templateId',
'fmt': 'H'
}, {
'name': 'schemaId',
'fmt': 'H'
}, {
'name': 'version',
'fmt': 'H'
}, {
'name': 'ClOrdID',
'fmt': 'Q'
}, {
'name': 'QueueSize',
'fmt': 'I'
}, {
'name': 'PenaltyRemain',
'fmt': 'I'
}]
Serializer.__init__(self, tag, fields)
def fit_implementation(project_name, version, registry_add):
path = Serializer().get_folder_name(project_name)
json = Serializer().read_model(path, version)
if not 'error' in json:
estimator, model, version = get_estimator(path, json, version)
messages = []
try:
producer = estimator.fit(model)
if inspect.isgenerator(producer):
for message in producer:
messages.append(message)
else:
messages = producer
Serializer().save_changes(path, version, model, add=False)
Serializer().save_fit_logs(path, version, messages)
if model.get_dashboard():
Serializer().set_dashboard(path, model.get_dashboard())
except:
return {'exception': traceback.format_exc()}
if model.registered_instance is not None:
registry_add(project_name, model, model.registered_instance)
return messages if messages is not None else {}
return {'error': True}
class Logger:
def __init__(self, db_connection, job_id=None):
self.log_serializer = Serializer(db_connection, job_log)
self.job_id = job_id
def set_job_id(self, job_id):
self.job_id = job_id
def log(self, summary, details='', level=LogLevel.INFO):
self.log_serializer.post({
'job_id': self.job_id,
'logged_at': datetime.now(),
'log_level': level,
'summary': summary,
'details': details,
})
def warn(self, summary, details=''):
self.log(summary, details, LogLevel.WARNING)
def info(self, summary, details=''):
self.log(summary, details, LogLevel.INFO)
def error(self, summary, details=''):
self.log(summary, details, LogLevel.ERROR)
def exception(self):
try:
etype, eval, tb = exc_info()
summary = str(eval)[:512]
details = traceback.format_exception(etype, eval, tb)
self.error(summary, details)
except Exception:
self.error('Exception occurred while catching exception.')
def __init__(self):
tag = 5001
fields = [{
'name': 'blockLength',
'fmt': 'H'
}, {
'name': 'templateId',
'fmt': 'H'
}, {
'name': 'schemaId',
'fmt': 'H'
}, {
'name': 'version',
'fmt': 'H'
}, {
'name': 'RequestTimestamp',
'fmt': 'Q'
}, {
'name': 'KeepaliveInterval',
'fmt': 'L'
}, {
'name': 'NextSeqNo',
'fmt': 'Q'
}]
Serializer.__init__(self, tag, fields)
def __init__(self, public_key, private_key):
# save inputs
self.private_key = private_key
self.public_key = public_key
self.public_key_string = public_key.exportKey().decode("utf-8")
# build wallet
self.wallet = Wallet(self.public_key_string)
# register miner
pseudonym = resolve_pseudonym(self.public_key_string)
if pseudonym == "":
pseudonym = input("\nChoose a pseudonym: ")
else:
print(
"\nThis key has already been registered.\nRegistered pseudonym: "
+ pseudonym)
self.miner = Miner(pseudonym, self.public_key_string)
self.s = Serializer()
# build menu
self.menu = "\nchoose an number:"
self.menu += "\n1.) mine"
self.menu += "\n2.) check balance"
self.menu += "\n3.) make transaction"
self.menu += "\nchoice: "
def getAudit(self):
try:
status = self.session.query(IsAudit).filter(IsAudit.id == 1).all()
status_maJia = self.session.query(IsAuditPro).filter(IsAuditPro.id == 1).all()
moments_control = self.session.query(MomentsControl).filter(MomentsControl.id == 1).all()
moments_control_android = self.session.query(MomentsControlAndroid).filter(MomentsControlAndroid.id == 1).all()
statusResult = Serializer(status, many=True).data
statusPro = Serializer(status_maJia, many=True).data
s_mc = Serializer(moments_control, many=True).data
s_mca = Serializer(moments_control_android, many=True).data
# iOS分享开关
iOSShareSwitch = s_mc[0]['control_status']
# 安卓分享开关
androidShareSwitch = s_mca[0]['control_status']
# iOS审核开关
iOSReviewSwitch = statusResult[0]['is_audit']
# iOS马甲包审核开关
iOSMaJiaReviewSwitch = statusPro[0]['is_audit']
allStatus = {
'isAudit':iOSReviewSwitch,
'iOSMaJiaAudit':iOSMaJiaReviewSwitch,
'iOSShare':iOSShareSwitch,
'androidShare':androidShareSwitch
}
# print allStatus
except:
self.session.rollback()
self.session.close()
return allStatus
def serialize_file(filename, action):
target_path = None
if action == 'encrypt':
target_path = Serializer.encrypt(filename)
elif action == 'decrypt':
target_path = Serializer.decrypt(filename)
return target_path
def __init__(self):
tag = 7011
fields = [{
'name': 'blockLength',
'fmt': 'H'
}, {
'name': 'templateId',
'fmt': 'H'
}, {
'name': 'schemaId',
'fmt': 'H'
}, {
'name': 'version',
'fmt': 'H'
}, {
'name': 'Timestamp',
'fmt': 'Q'
}, {
'name': 'TradingSessionID',
'fmt': 'i'
}, {
'name': 'TradSesEvent',
'fmt': 'B'
}]
Serializer.__init__(self, tag, fields)
def updateUTXO():
db = TinyDB('db/blk.json')
lstBlk = (db.all()[-1:])[0]['Transactions']
dlt = []
apd = []
for i in lstBlk:
tmp = Param(i)
obj = Serializer(tmp)
seri = obj.make()
hash = (sha256(sha256(
binascii.unhexlify(seri)).digest()).digest()[::-1]).hex()
for y in i['tx_in']:
if y['Previous txid'] != "0000000000000000000000000000000000000000000000000000000000000000":
dlt.append({
'txid': y['Previous txid'],
'index': y['Previous Tx Index']
})
g = 0
for z in i['tx_out']:
apd.append({
'value': z['value'],
'txid': hash,
'index': g,
'Public_Script': z['Public Script']
})
g = g + 1
deleteUTXO(dlt)
appendUTXO(apd)
def __init__(self):
tag = 6002
fields = [{
'name': 'blockLength',
'fmt': 'H'
}, {
'name': 'templateId',
'fmt': 'H'
}, {
'name': 'schemaId',
'fmt': 'H'
}, {
'name': 'version',
'fmt': 'H'
}, {
'name': 'ClOrdID',
'fmt': 'Q'
}, {
'name': 'OrderID',
'fmt': 'q'
}, {
'name': 'Account',
'fmt': '7s'
}]
Serializer.__init__(self, tag, fields)
def prepare_different_items_list(self, rest_client, headers):
serializer = Serializer()
url = self.config["global"]["xml_url"]
xml = rest_client.send_get_binary(url, None, None)
read_item_list = list()
if xml[1] != 200:
logging.info("No file found under the URI: %s, exiting now..." %
str(url))
exit(-1)
root_item = serializer.deserialize_xml(str(xml[0], "utf8"))
commodities = root_item
magento_product_list = list(self.get_product_list(headers)[0]["items"])
magento_product_dict = dict()
for magento_product in magento_product_list:
magento_product_dict[magento_product["sku"]] = magento_product
for commodity in commodities:
if commodity[1].text:
if commodity[1].text in magento_product_dict:
magento_product = magento_product_dict[commodity[1].text]
result_magento_product = self.link_products(
magento_product, commodity)
if result_magento_product:
read_item_list.append(result_magento_product)
else:
logging.warning(
"There is no product of sku {0} in Magento database.".
format(commodity[1].text))
return read_item_list
def __init__(self):
tag = 5008
fields = [{
'name': 'blockLength',
'fmt': 'H'
}, {
'name': 'templateId',
'fmt': 'H'
}, {
'name': 'schemaId',
'fmt': 'H'
}, {
'name': 'version',
'fmt': 'H'
}, {
'name': 'ClOrdID',
'fmt': 'Q'
}, {
'name': 'RefTagID',
'fmt': 'I'
}, {
'name': 'SessionRejectReason',
'fmt': 'B'
}]
Serializer.__init__(self, tag, fields)
def start_workflow(shared_state, start_date, review_number=0):
db_connection = setup_db().connect()
logger = Logger(db_connection)
shared_state.job_id = None
shared_state.completed = False
max_downloads = environ.get('MAX_DOWNLOADS')
if max_downloads is not None:
max_downloads = int(max_downloads)
max_upload_workers = int(environ.get('MAX_UPLOADERS', 20))
try:
workflow = Workflow(
db_connection, logger,
start_date,
max_downloads, max_upload_workers,
environ.get('ALLOW_REPEAT', 'FALSE') == 'TRUE'
)
workflow.start(shared_state)
except Exception:
logger.exception()
if shared_state.job_id is not None:
job_serializer = Serializer(db_connection, job)
job_serializer.put(shared_state.job_id, {
'status': JobStatus.FAILED,
})
def __init__(self):
tag = 5005
fields = [{
'name': 'blockLength',
'fmt': 'H'
}, {
'name': 'templateId',
'fmt': 'H'
}, {
'name': 'schemaId',
'fmt': 'H'
}, {
'name': 'version',
'fmt': 'H'
}, {
'name': 'NextSeqNo',
'fmt': 'Q'
}, {
'name': 'RequestTimestamp',
'fmt': 'Q'
}, {
'name': 'Count',
'fmt': 'I'
}]
Serializer.__init__(self, tag, fields)
def __init__(self):
tag = 5000
fields = [{
'name': 'blockLength',
'fmt': 'H'
}, {
'name': 'templateId',
'fmt': 'H'
}, {
'name': 'schemaId',
'fmt': 'H'
}, {
'name': 'version',
'fmt': 'H'
}, {
'name': 'Timestamp',
'fmt': 'Q'
}, {
'name': 'KeepaliveInterval',
'fmt': 'L'
}, {
'name': 'Credentials',
'fmt': '20s'
}]
Serializer.__init__(self, tag, fields)
def blockReturnHash():
hash = request.form['data']
db = TinyDB('db/blk.json')
lst = db.all()
response = app.response_class(response=json.dumps({"status": "not find"}),
status=404,
mimetype='application/json')
for i in lst:
data = sha256(
((i["Block Header"]['Version']).to_bytes(4, "little") +
binascii.unhexlify(i["Block Header"]['Previous Block Hash']) +
binascii.unhexlify(i["Block Header"]['Merkle Root']) +
(i["Block Header"]['Timestamp']).to_bytes(4, "little") +
(binascii.unhexlify(
i["Block Header"]["Difficulty Target"])[::-1]) +
(i["Block Header"]['Nonce']).to_bytes(4, "little"))).hexdigest()
if data == hash:
tranHashLst = []
for y in i['Transactions']:
tmp = Param(y)
obj = Serializer(tmp)
data = sha256(binascii.unhexlify(obj.make())).hexdigest()
tranHashLst.append(data)
i["Transaction Data"] = tranHashLst
response = app.response_class(response=json.dumps(i),
status=200,
mimetype='application/json')
break
return response
def expectRepeat(self, tokens, level=0, debug=False):
yield Serializer(True, [], None)
# if we get to this point we need at least one instance of this pattern.
index = tokens.getIndex()
exhausted = tokens.isExhausted()
for value in self.children[0].expect(tokens, level + 1, debug):
if value and (not tokens.isExhausted()):
newIndex = tokens.getIndex()
newExhaust = tokens.isExhausted()
tValue = value.transform(lambda x: [x])
# create a new instance of the repeat pattern
# since the first thing this does is yield True,
# we don't do it here.
for newValue in self.expectRepeat(tokens, level, debug):
if newValue:
tNewValue = newValue.transform(
lambda x: tValue.getArgs() + x)
yield tNewValue
tokens.setIndex(newIndex, newExhaust)
elif value and tokens.isExhausted():
tValue = value.transform(lambda x: [x])
yield tValue
tokens.setIndex(index, exhausted)
err = "Unknown Error in expectRepeat"
if not value:
err = value.getError()
else:
if not newValue:
err = newValue.getError()
yield Serializer(False, None, err)
def __init__(self):
tag = 7006
fields = [{
'name': 'blockLength',
'fmt': 'H'
}, {
'name': 'templateId',
'fmt': 'H'
}, {
'name': 'schemaId',
'fmt': 'H'
}, {
'name': 'version',
'fmt': 'H'
}, {
'name': 'ClOrdID',
'fmt': 'Q'
}, {
'name': 'Timestamp',
'fmt': 'Q'
}, {
'name': 'OrdRejReason',
'fmt': 'i'
}]
Serializer.__init__(self, tag, fields)
def __init__(self, problem, config):
self.abs_path = os.path.dirname(os.path.abspath(__file__))
self.serializer = Serializer()
self.config_file = config
self.config = self.serializer.read_config(self.abs_path + "/" + config)
self.ik_solution_generator = IKSolutionGenerator()
self.clear_stats(problem)
self.run(problem)
class Collect:
def __init__(self):
cv2.namedWindow('stimulus', cv2.WND_PROP_FULLSCREEN)
cv2.setWindowProperty('stimulus', cv2.WND_PROP_FULLSCREEN,
cv2.WINDOW_FULLSCREEN)
self.serializer = Serializer(int(os.getenv("EBC_SUBJECT_ID")),
int(os.getenv("EBC_SESSION_ID")))
def __del__(self):
cv2.destroyWindow('stimulus')
def run(self):
with Manager() as manager:
capture_index = manager.Value(int, value=0)
with PicutureCapture(
float(os.getenv("EBC_PICTURE_CAPTURE_INTERVAL")),
os.getenv("EBC_PICTURE_CAPTURE_OUTPUT_PATH"),
capture_index, manager.Event(),
manager.Event()) as picture_capture:
with SignalCapture(os.getenv("EBC_SIGNAL_PORT"),
float(os.getenv("EBC_SIGNAL_BOUND_RATE")),
manager.list(), 0, capture_index,
manager.Event(),
manager.Event()) as signal_capture:
stimulus = Stimulus(
int(os.getenv("EBC_STIMULUS_EXPOSITION_TIME")),
os.getenv("EBC_STIMULUS_IAPS_PATH"),
list(
map(
str,
json.loads(
os.getenv("EBC_STIMULUS_SELECTED_SET")))))
sam = SelfAssessmentManikin(
int(os.getenv('EBC_SAM_TIME_LIMIT', "15")))
for i, picture in enumerate(stimulus.stimulus_list):
capture_index.value = i
logger.info('Capture {} - Picture {}'.format(
i, picture))
signal_capture.start_capture()
picture_capture.start_capture()
stimulus.exhibit_stimulus()
picture_capture.stop_capture()
signal_capture.stop_capture()
signals = signal_capture.pop_buffer()
try:
sam.apply_test()
except ValueError:
logger.warning(
"Incorrect response in SAM, missed capture {:04}",
capture_index.value)
logger.info("{:03} Singnals: {}", len(signals),
signals)
logger.info("Valence: {} | Arousal: {}", sam.valence,
sam.arousal)
self.serializer.serialize(capture_index.value,
stimulus.stimulus,
sam.valence, sam.arousal,
signals)
def save_to_file(self):
self.update_data()
try:
save_file_path = QtWidgets.QFileDialog.getSaveFileName(
self, 'Save file...', os.path.dirname(os.getcwd()))[0]
Serializer.serialize(save_file_path, self.data)
except OSError:
QtWidgets.QMessageBox.critical(
self, 'Error', 'Во время сохранения файла произошла ошибка')
def __init__(self, pseudonym, public_key):
self.public_key = public_key
self.serializer = Serializer()
data = {
"pseudonym": pseudonym,
"public_key": public_key,
}
r = requests.post("http://localhost:5000/register_miner", data=data)
if r.text != "":
raise ValueError(r.text)
def on_get(self, request, response):
if not check_token(request):
response.media = {'error': True, 'message': 'Not authorized'}
return
data = json.loads(request.stream.read().decode())
result = Serializer().read_project(Serializer().get_folder_name(
data['project'] if 'project' in data else ''
))
response.media = {"predictions_model": result['predictions_model']} \
if 'predictions_model' in result else result
def play(self):
"""
Function where whole game is happening.
:returns: None.
:rtype: None.
"""
custom_log.logger.info(
"---------------------------------------------------")
while True:
valid_direction = self.game_map.get_valid_directions()
custom_log.logger.info(
"Input 'save'/'load' to save/load the game.")
custom_log.logger.info(f"Valid directions - {valid_direction}")
self.player.check_status(self.game_map)
self.player.print_status()
self.game_map.print_map()
direction = DungeonInput.get_direction()
if direction == "save":
Serializer.save_map(self.game_map)
continue
elif direction == "load":
new_game_map = Serializer.load_map()
if new_game_map == "":
custom_log.logger.error("Failed to Load Game")
continue
self.game_map = new_game_map
continue
if direction not in valid_direction:
custom_log.logger.warning("Can't move there!")
continue
self.player.move(self.game_map, direction)
custom_log.logger.info(
"---------------------------------------------------")
if self.player.hp == 0:
self.end_game()
break
elif self.player.bag == Game.TREASURE_TO_WIN:
self.win_game()
break
def __init__(self):
tag = 7008
fields = [{
'name': 'blockLength',
'fmt': 'H'
}, {
'name': 'templateId',
'fmt': 'H'
}, {
'name': 'schemaId',
'fmt': 'H'
}, {
'name': 'version',
'fmt': 'H'
}, {
'name': 'ClOrdID',
'fmt': 'Q'
}, {
'name': 'Timestamp',
'fmt': 'Q'
}, {
'name': 'OrderID',
'fmt': 'q'
}, {
'name': 'TrdMatchID',
'fmt': 'q'
}, {
'name': 'Flags',
'fmt': 'q'
}, {
'name': 'LastPx',
'fmt': 'q'
}, {
'name': 'LastQty',
'fmt': 'I'
}, {
'name': 'OrderQty',
'fmt': 'I'
}, {
'name': 'TradingSessionID',
'fmt': 'i'
}, {
'name': 'ClOrdLinkID',
'fmt': 'i'
}, {
'name': 'SecurityID',
'fmt': 'i'
}, {
'name': 'Side',
'fmt': 'B'
}]
Serializer.__init__(self, tag, fields)
def test_serializer_for_no_chinese_split():
text1 = "\nI\'m his pupp\'peer, and i have a ball\t"
text2 = '\t叫Stam一起到nba打篮球\n'
text3 = '\n\n现在时刻2014-04-08\t\t'
serializer = Serializer(do_chinese_split=False)
serial_text1 = serializer.serialize(text1)
serial_text2 = serializer.serialize(text2)
serial_text3 = serializer.serialize(text3)
assert serial_text1 == ['i', "'", 'm', 'his', 'pupp', "'", 'peer', ',', 'and', 'i', 'have', 'a', 'ball']
assert serial_text2 == ['叫', 'stam', '一', '起', '到', 'nba', '打', '篮', '球']
assert serial_text3 == ['现', '在', '时', '刻', '2014', '-', '04', '-', '08']
class Calculator:
def __init__(self, side_length):
self._serializer = Serializer(side_length)
self._file_name = "serializer.bin"
@property
def serializer(self):
return self._serializer
def calculate_sum_area(self):
sum_areas = 0
for figure in self._serializer.figures_list:
figure.calc_area()
sum_areas += figure.area
self._serializer.sum_areas = round(sum_areas)
def save(self):
with open(self._file_name, "wb") as file:
file.write(self._serializer.serialize())
file.flush()
file.close()
def restore(self):
with open(self._file_name, "rb") as file:
self._serializer = Serializer.deserialize(file.read())
file.close()
def show(self):
self._serializer.print_objects()
def update_figures(self, side_length):
self._serializer.update_figures(side_length)
if self._serializer.sum_areas != 0:
self.calculate_sum_area()
@staticmethod
def binary_to_integer(binary_value):
"""
This function converts binary value as a string to decimal integer value
:param binary_value: string binary value
:return: decimal integer value
"""
integer_value = 0
binary_index = 0
for char in binary_value[::-1]:
if char == '1':
integer_value += pow(2, binary_index)
binary_index += 1
return int(integer_value)
def __init__(self, device, dataset, verbose, save, save_aux):
self.dataset_name = dataset
self.train_loader, self.valid_loader = get_data(dataset=dataset)
self.device = device
self.serializer = Serializer()
self.target = 'reconstruction'
self.trainer_basis = None
self.writer = None
self.verbose = verbose
self.save = save
self.save_aux = save_aux
self.model_name = None
self.config_dict = None
self.transformer = None
def check_serialized_name(self, name, expected_indexes_with_ones):
serializer = Serializer(['Germany', 'France'])
serialized_example = serializer.serialize_example(name)
if expected_indexes_with_ones is None:
self.assertIsNone(serialized_example)
else:
self.assertEqual(len(expected_indexes_with_ones), len(serialized_example))
for i in range(len(expected_indexes_with_ones)):
serialized_char = serialized_example[i]
self.assertEqual(sum(serialized_char), 1.0)
self.assertEqual(serialized_char[expected_indexes_with_ones[i]], 1)
def record_voice():
# globals
FORMAT = pyaudio.paInt16
CHANNELS = 1
RATE = 44100
CHUNK = 1024
RECORD_SECONDS = 3
button = Button(27)
led = LED(18)
# start recording
audio = pyaudio.PyAudio()
stream = audio.open(format=FORMAT,
channels=CHANNELS,
rate=RATE,
input=True,
frames_per_buffer=CHUNK)
# print("recording...")
frames = []
# record for RECORD_SECONDS
button.wait_for_press()
led.on()
while button.is_pressed:
# for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
data = stream.read(CHUNK, exception_on_overflow=False)
frames.append(data)
led.off()
# print("finished recording")
# stop Recording
stream.stop_stream()
stream.close()
audio.terminate()
# write your new .wav file with built in Python 3 Wave module
waveFile = wave.open("out.wav", 'wb')
waveFile.setnchannels(CHANNELS)
waveFile.setsampwidth(audio.get_sample_size(FORMAT))
waveFile.setframerate(RATE)
waveFile.writeframes(b''.join(frames))
waveFile.close()
# stt
s = Serializer()
with open("out.wav", "rb") as fin:
r = requests.post(url="http://192.168.1.172:8000/transcribe",
data={"audio": s.serialize(fin.read())})
print(r.json())
return r.json()["msg"]
class RPCStub(object):
def __init__(self, uri, stub, namespace=None):
self.uri = uri
self.stub = stub
self.namespace = namespace
self._seril = Serializer(namespace)
self.logger = None
def process(self, method, data):
ret = exception = None
try:
args, kw = self._seril.deserialize(data)
try:
self.logger and self.logger.info(u"method:%s, args:%s, kw:%s" % (method,
args, kw))
except Exception, e:
#@todo: fix the decode problem.
self.logger and self.logger.info(str(e))
h = self._local_service(self.stub, method)
if h is None: raise RPCException(u"Not found interface '%s'" % method)
ret = h(*args, **kw)
self.logger and self.logger.info("return:%s" % (ret, ))
except BaseException, e:
exception = e
self.logger and self.logger.exception(e)
class NodeMaker:
def __init__(self):
self.ser = Serializer()
self.words = defaultdict(int)
self.graph = Graph()
def PosNo(self, morph):
for i, p in enumerate([u'形容詞', u'名詞']):
if p in morph.pos():
return i+2
return 0
def regist(self, text):
lines = text.split('\n')
lst = []
for lnum, line in enumerate(lines):
morphs = wakachi.parse(text)
for morph in morphs:
if self.PosNo(morph):
lst.append(morph)
self.words[(morph.posid, morph.original)] += 1
else:
lst.append(None)
lst += [None]*5
if line == '':
self.consume(lst)
lst = []
self.consume(lst)
def consume(self, lst, back=3, fore=10): #0:N, 1:V, 2:Y
size = len(lst)
for i in xrange(size):
if lst[i] is None: continue
posno = self.PosNo(lst[i])
node = []
for x in xrange(posno):
node.append(self.ser.encode((lst[i].posid, lst[i].original(), x)))
self.graph.registerNode(node[x])
#for node = V
for j in xrange(max(0,i-fore), min(size,i+back)):
if lst[j] is None or self.PosNo(lst[j]) == 2: continue
ny = self.ser.encode((lst[j].posid, lst[j].original(), 2))
self.graph.addEdge(node[1], ny)
#for node = Y
if posno == 3:
for j in xrange(max(0,i-back), min(size,i+fore)):
if lst[j] is None: continue
nv = self.ser.encode((lst[j].posid, lst[j].original(), 1))
self.graph.addEdge(node[2],nv)
def __init__(self, problem, config):
self.abs_path = os.path.dirname(os.path.abspath(__file__))
self.serializer = Serializer()
self.config_file = config
self.config = self.serializer.read_config(self.abs_path + "/" + config)
self.ik_solution_generator = IKSolutionGenerator()
self.clear_stats(problem)
self.run(problem)
def _fetch_metadata_by_json(self, json_obj):
""" parses incoming json object representation and fetches related
object metadata from the server. Returns None or the Metadata object.
This method is called only when core object is not found in cache, and
if metadata should be requested anyway, it could be done faster with
this method that uses GET /neo/<obj_type>/198272/metadata/
Currently not used, because inside the pull function (in cascade mode)
it may be faster to collect all metadata after the whole object tree is
fetched. For situations, when, say, dozens of of objects tagged with the
same value are requested, it's faster to fetch this value once at the
end rather than requesting related metadata for every object.
"""
if not json_obj['fields'].has_key('metadata') or \
not json_obj['fields']['metadata']:
return None # no metadata field or empty metadata
url = json_obj['permalink']
# TODO move requests to the remote class
resp = requests.get( url + 'metadata' , cookies=self._meta.cookie_jar )
raw_json = get_json_from_response( resp )
if not resp.status_code == 200:
message = '%s (%s)' % (raw_json['message'], raw_json['details'])
raise errors.error_codes[resp.status_code]( message )
if not raw_json['metadata']: # if no objects exist return empty result
return None
mobj = Metadata()
for p, v in raw_json['metadata']:
prp = Serializer.deserialize(p, self)
val = Serializer.deserialize(v, self)
prp.append( val )
# save both objects to cache
self._cache.add_object( prp )
self._cache.add_object( val )
setattr( mobj, prp.name, prp )
return mobj # Metadata object with list of properties (tags)
def __init__(self, uri, method=None, namespaces=None, sid=''):
self.__serviceURL = uri
self.__serviceName = method
self.namespaces = isinstance(namespaces, ClassLoader) and namespaces or \
ClassLoader(namespaces or [])
self._seril = Serializer(self.namespaces)
self.sid = sid
self.logger = None
self.start_call_listener = []
self.end_call_listener = []
def add(self, product_id, unit_cost, quantity=1, **extra_data_dict):
"""
Returns True if the addition of the product of the given product_id and unit_cost with given quantity
is succesful else False.
Can also add extra details in the form of dictionary.
"""
product_dict = self.__product_dict(
unit_cost, quantity, extra_data_dict)
self.redis_connection.hset(
self.user_redis_key, product_id, Serializer.dumps(product_dict))
self.user_cart_exists = self.cart_exists(self.user_id)
self.set_ttl()
def get_product(self, product_id):
"""
Returns the cart details as a Dictionary for the given product_id
"""
if self.user_cart_exists:
product_string = self.redis_connection.hget(
self.user_redis_key, product_id)
if product_string:
return Serializer.loads(product_string)
else:
return {}
else:
raise ErrorMessage("The user cart is Empty")
def write(self, buf, offset=0):
"""
Updates object information
:param buf: a YAML representation of an object.
:type buf: str
:return: 0 on success or and negative error code.
"""
try:
new = Serializer.deserialize(self.model_instance.__class__, buf)
new.id = self.model_instance.id
except Exception, e:
return -1 # TODO find a way to handle expceptions better..
def __init__(self, model, **kwargs):
self.model = model
self.fields = kwargs.get("fields") or self.model._meta.get_all_field_names()
self.name = kwargs.get("name")
self.parent_name = kwargs.get("parent_name", False)
if self.parent_name:
self.parent_id = self.parent_name + "_id"
else:
self.parent_id = ""
self.id = self.name + "_id"
self.set_name = self.name + "_set"
self.relations = kwargs.get("relations") or [
rel.get_accessor_name() for rel in model._meta.get_all_related_objects()
]
self.parent_model = kwargs.get("parent_model")
self.serialize = Serializer(self.fields)
def __push_data_from_obj(self, obj):
""" saves array data to disk in HDF5s and uploads new datafiles to the
server according to the arrays of the given obj. Saves datafile objects
to cache """
data_refs = {} # collects all references to the related data - output
model_name = self._meta.get_type_by_obj( obj )
data_attrs = self._meta.get_array_attr_names( model_name )
attrs_to_sync = self._cache.detect_changed_data_fields( obj )
for attr in data_attrs: # attr is like 'times', 'signal' etc.
arr = None
if attr in attrs_to_sync:
# 1. get current array and units
fname = self._meta.app_definitions[model_name]['data_fields'][attr][2]
if fname == 'self':
arr = obj # some NEO objects like signal inherit array
else:
arr = getattr(obj, fname)
if not type(arr) == type(None): # because of NEO __eq__
units = Serializer.parse_units(arr)
datapath = self._cache.save_data(arr)
json_obj = self._remote.save_data(datapath)
# update cache data map
datalink = json_obj['permalink']
fid = str(get_id_from_permalink( datalink ))
folder, tempname = os.path.split(datapath)
new_path = os.path.join(folder, fid + tempname[tempname.find('.'):])
os.rename(datapath, new_path)
self._cache.update_data_map(fid, new_path)
data_refs[ attr ] = {'data': datalink, 'units': units}
else:
data_refs[ attr ] = None
return data_refs
def fix_peewee_obj(obj):
"""
Serializes a single peewee object
"""
ser = Serializer()
return ser.serialize_object(obj)
def __init__(self, selfNodeAddr, otherNodesAddrs, conf=None):
if conf is None:
self.__conf = SyncObjConf()
else:
self.__conf = conf
self.__selfNodeAddr = selfNodeAddr
self.__otherNodesAddrs = otherNodesAddrs
self.__unknownConnections = {} # descr => _Connection
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftCurrentTerm = 0
self.__votesCount = 0
self.__raftLeader = None
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__raftLog = [] # (command, logID, term)
self.__raftLog.append((None, 1, self.__raftCurrentTerm))
self.__raftCommitIndex = 1
self.__raftLastApplied = 1
self.__raftNextIndex = {}
self.__raftMatchIndex = {}
self.__lastSerializedTime = time.time()
self.__forceLogCompaction = False
self.__socket = None
self.__resolver = DnsCachingResolver(self.__conf.dnsCacheTime, self.__conf.dnsFailCacheTime)
self.__serializer = Serializer(self.__conf.fullDumpFile, self.__conf.logCompactionBatchSize)
self.__poller = createPoller()
self.__isInitialized = False
self.__lastInitTryTime = 0
self._methodToID = {}
self._idToMethod = {}
methods = sorted([m for m in dir(self) if callable(getattr(self, m))])
for i, method in enumerate(methods):
self._methodToID[method] = i
self._idToMethod[i] = getattr(self, method)
self.__thread = None
self.__mainThread = None
self.__initialised = None
self.__commandsQueue = Queue.Queue(self.__conf.commandsQueueSize)
self.__nodes = []
self.__newAppendEntriesTime = 0
self.__commandsWaitingCommit = collections.defaultdict(list) # logID => [(termID, callback), ...]
self.__commandsLocalCounter = 0
self.__commandsWaitingReply = {} # commandLocalCounter => callback
self.__properies = set()
for key in self.__dict__:
self.__properies.add(key)
if self.__conf.autoTick:
self.__mainThread = threading.current_thread()
self.__initialised = threading.Event()
self.__thread = threading.Thread(target=SyncObj._autoTickThread, args=(weakref.proxy(self),))
self.__thread.start()
while not self.__initialised.is_set():
pass
else:
self.__initInTickThread()
def __init__(self, uri, stub, namespace=None):
self.uri = uri
self.stub = stub
self.namespace = namespace
self._seril = Serializer(namespace)
self.logger = None
def __init__(self):
Serializer.__init__(self)
def __init__(self, save):
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
if not os.path.isdir("stats"):
os.makedirs("stats")
self.save = save
self.utils = Utils()
logging.info("Loading cartesian coordinates...")
serializer = Serializer()
rave_env = Environment()
f = "stats/model/test.urdf"
rave_env.initializeEnvironment(f)
self.link_dimensions = rave_env.getRobotLinkDimensions()
rave_env.destroy()
self.setup_kinematics(serializer, "stats")
self.process_covariances = []
for logfile in glob.glob("stats" + "/*.log"):
self.process_covariances.append(serializer.read_process_covariance(logfile))
#arr = serializer.deserialize_joint_angles(path="stats", file="state_path1.txt")
#self.plot_state_path(serializer, arr)
#sleep()
self.create_video(serializer, "stats")
logging.info("plotting paths")
try:
self.plot_paths(serializer)
except:
logging.warn("Paths could not be plotted")
logging.info("Reading config")
config = serializer.read_config("config.yaml", path="stats")
#try:
#self.plot_end_effector_paths(serializer, plot_scenery=True, plot_manipulator=True)
#except:
# print "Error. End effector paths could not be plotted"
logging.info("Plotting mean planning times")
self.plot_mean_planning_times(serializer,
dir="stats",
filename="mean_planning_times_per_step*.yaml",
output="mean_planning_times_per_step.pdf")
self.plot_mean_planning_times(serializer,
dir="stats",
filename="mean_planning_times_per_run*.yaml",
output="mean_planning_times_per_run.pdf")
self.plot_particles(serializer, particle_limit=config['particle_plot_limit'])
try:
self.plot_paths(serializer, best_paths=True)
except:
logging.warn("Best_paths could not be plotted")
try:
cart_coords = serializer.load_cartesian_coords(path="stats")
except:
logging.warn("Cartesian_coords could not be plotted")
logging.info("Plotting average distance to goal" )
try:
self.plot_average_dist_to_goal(serializer, cart_coords)
except:
logging.warn("Average distance to goal could not be plotted")
logging.info("plotting mean rewards")
try:
self.plot_mean_rewards(serializer)
except Exception as e:
logging.warn("Mean rewards could not be plotted: " + str(e))
logging.info("Plotting EMD graph...")
try:
self.plot_emd_graph(serializer, cart_coords)
except:
logging.warn("EMD graphs could not be plotted")
logging.info("Plotting histograms...")
try:
self.save_histogram_plots(serializer, cart_coords)
except:
logging.warn("Histograms could not be plotted")
def read(self, size=-1, offset=0):
"""
Returns an attached object representation as YAML file (bytestring).
"""
return Serializer.serialize(self.model_instance) # binary?
class LQG:
def __init__(self, show_scene, deserialize, append_paths):
self.abs_path = os.path.dirname(os.path.abspath(__file__))
""" Reading the config """
warnings.filterwarnings("ignore")
self.init_serializer()
config = self.serializer.read_config("config_lqg.yaml", path=self.abs_path)
self.set_params(config)
if self.seed < 0:
"""
Generate a random seed that will be stored
"""
self.seed = np.random.randint(0, 4294967)
np.random.seed(self.seed)
logging_level = logging.WARN
if config['verbose']:
logging_level = logging.DEBUG
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging_level)
np.set_printoptions(precision=16)
dir = self.abs_path + "/stats/lqg"
tmp_dir = self.abs_path + "/tmp/lqg"
self.utils = Utils()
if not self.init_robot(self.robot_file):
logging.error("LQG: Couldn't initialize robot")
return
if not self.setup_scene(self.environment_file, self.robot):
return
#self.show_state_distribution(urdf_model_file, environment_file)
#self.check_continuous_collide(self.robot_file, self.environment_file)
if show_scene:
self.test_sensor(self.robot_file, self.environment_file, self.abs_path + "/model/sensor.xml")
self.run_viewer(self.robot_file, self.environment_file)
self.clear_stats(dir)
logging.info("Start up simulator")
sim = Simulator()
path_evaluator = PathEvaluator()
path_planner = PathPlanningInterface()
logging.info("LQG: Generating goal states...")
goal_states = get_goal_states("lqg",
self.abs_path,
self.serializer,
self.obstacles,
self.robot,
self.max_velocity,
self.delta_t,
self.start_state,
self.goal_position,
self.goal_radius,
self.planning_algortihm,
self.path_timeout,
self.num_generated_goal_states,
self.continuous_collision,
self.environment_file,
self.num_cores)
if len(goal_states) == 0:
logging.error("LQG: Couldn't generate any goal states. Problem seems to be infeasible")
return
logging.info("LQG: Generated " + str(len(goal_states)) + " goal states")
sim.setup_reward_function(self.discount_factor, self.step_penalty, self.illegal_move_penalty, self.exit_reward)
path_planner.setup(self.robot,
self.obstacles,
self.max_velocity,
self.delta_t,
self.use_linear_path,
self.planning_algortihm,
self.path_timeout,
self.continuous_collision,
self.num_cores)
if self.dynamic_problem:
path_planner.setup_dynamic_problem(self.robot_file,
self.environment_file,
self.simulation_step_size,
self.num_control_samples,
self.min_control_duration,
self.max_control_duration,
self.add_intermediate_states,
self.rrt_goal_bias,
self.control_sampler)
path_planner.set_start_and_goal(self.start_state, goal_states, self.goal_position, self.goal_radius)
A, H, B, V, W, C, D, M_base, N_base = self.problem_setup(self.delta_t, self.robot_dof)
time_to_generate_paths = 0.0
if check_positive_definite([C, D]):
m_covs = None
if self.inc_covariance == "process":
m_covs = np.linspace(self.min_process_covariance,
self.max_process_covariance,
self.covariance_steps)
elif self.inc_covariance == "observation":
m_covs = np.linspace(self.min_observation_covariance,
self.max_observation_covariance,
self.covariance_steps)
emds = []
mean_planning_times = []
time_to_generate_paths = 0.0
paths = []
if ((not append_paths) and deserialize):
paths = self.serializer.deserialize_paths(self.abs_path + "/paths.txt", self.robot_dof)
#paths = [paths[26]]
if len(paths) == 0:
print "LQG: Generating " + str(self.num_paths) + " paths from the inital state to the goal position..."
t0 = time.time()
paths = path_planner.plan_paths(self.num_paths, 0)
if len(paths) == 0:
logging.error("LQG: Couldn't create any paths within the given time.")
return
time_to_generate_paths = time.time() - t0
print "LQG: Time to generate paths: " + str(time_to_generate_paths) + " seconds"
if self.plot_paths:
self.serializer.save_paths(paths, "paths.yaml", self.overwrite_paths_file, path=dir)
if deserialize or append_paths:
self.serializer.serialize_paths(self.abs_path + "/paths.txt", paths, append_paths, self.robot_dof)
paths = self.serializer.deserialize_paths(self.abs_path + "/paths.txt", self.robot_dof)
""" Determine average path length """
avg_path_length = self.get_avg_path_length(paths)
self.serializer.save_avg_path_lengths(avg_path_length, path=dir)
cart_coords = []
best_paths = []
all_rewards = []
successes = []
for j in xrange(len(m_covs)):
print "LQG: Evaluating " + str(len(paths)) + " paths for covariance value " + str(m_covs[j]) + "..."
M = None
N = None
if self.inc_covariance == "process":
""" The process noise covariance matrix """
M = self.calc_covariance_value(self.robot, m_covs[j], M_base)
#M = m_covs[j] * M_base
""" The observation error covariance matrix """
N = self.calc_covariance_value(self.robot,
self.min_observation_covariance,
N_base,
covariance_type='observation')
elif self.inc_covariance == "observation":
M = self.calc_covariance_value(self.robot,
self.min_process_covariance,
M_base)
N = self.calc_covariance_value(self.robot,
m_covs[j],
N_base,
covariance_type='observation')
P_t = np.array([[0.0 for k in xrange(2 * self.robot_dof)] for l in xrange(2 * self.robot_dof)])
path_evaluator.setup(A, B, C, D, H, M, N, V, W,
self.robot,
self.sample_size,
self.obstacles,
self.goal_position,
self.goal_radius,
self.show_viewer_evaluation,
self.robot_file,
self.environment_file,
self.num_cores)
if self.dynamic_problem:
path_evaluator.setup_dynamic_problem()
path_evaluator.setup_reward_function(self.step_penalty, self.illegal_move_penalty, self.exit_reward, self.discount_factor)
t0 = time.time()
(path_index,
xs,
us,
zs,
control_durations,
objective,
state_covariances,
deviation_covariances,
estimated_deviation_covariances) = path_evaluator.evaluate_paths(paths, P_t, 0)
te = time.time() - t0
print "LQG: Time to evaluate " + str(len(paths)) + " paths: " + str(te) + "s"
mean_planning_time = time_to_generate_paths + te
print "LQG: Best objective value: " + str(objective)
print "LQG: Length of best path: " + str(len(xs))
print "LQG: Best path has index " + str(path_index)
best_paths.append([[xs[i] for i in xrange(len(xs))],
[us[i] for i in xrange(len(us))],
[zs[i] for i in xrange(len(zs))],
[control_durations[i] for i in xrange(len(control_durations))]])
sim.setup_problem(A, B, C, D, H, V, W, M, N,
self.robot,
self.enforce_control_constraints,
self.obstacles,
self.goal_position,
self.goal_radius,
self.max_velocity,
self.show_viewer_simulation,
self.robot_file,
self.environment_file)
if self.dynamic_problem:
sim.setup_dynamic_problem(self.simulation_step_size)
successes = 0
num_collisions = 0
rewards_cov = []
final_states= []
num_steps = 0
collided_num = 0
print "LQG: Running " + str(self.num_simulation_runs) + " simulations..."
for k in xrange(self.num_simulation_runs):
self.serializer.write_line("log.log", tmp_dir, "RUN #" + str(k + 1) + " \n")
print "simulation run: " + str(k)
(x_true,
x_tilde,
x_tilde_linear,
x_estimate,
P_t,
current_step,
total_reward,
terminal,
estimated_s,
estimated_c,
history_entries) = sim.simulate_n_steps(xs, us, zs,
control_durations,
xs[0],
np.array([0.0 for i in xrange(2 * self.robot_dof)]),
np.array([0.0 for i in xrange(2 * self.robot_dof)]),
xs[0],
np.array([[0.0 for k in xrange(2 * self.robot_dof)] for l in xrange(2 * self.robot_dof)]),
0.0,
0,
len(xs) - 1,
deviation_covariances,
estimated_deviation_covariances)
if terminal:
successes += 1
rewards_cov.append(total_reward)
#n, min_max, mean, var, skew, kurt = scipy.stats.describe(np.array(rewards_cov))
collided = False
for l in xrange(len(history_entries)):
history_entries[l].set_estimated_covariance(state_covariances[l])
history_entries[l].serialize(tmp_dir, "log.log")
if history_entries[l].collided:
num_collisions += 1
collided = True
if collided:
collided_num += 1
num_steps += history_entries[-1].t
final_states.append(history_entries[-1].x_true)
self.serializer.write_line("log.log", tmp_dir, "Reward: " + str(total_reward) + " \n")
self.serializer.write_line("log.log", tmp_dir, "\n")
""" Calculate the distance to goal area
"""
ee_position_distances = []
for state in final_states:
joint_angles = v_double()
joint_angles[:] = [state[y] for y in xrange(len(state) / 2)]
ee_position = v_double()
self.robot.getEndEffectorPosition(joint_angles, ee_position)
ee_position = np.array([ee_position[y] for y in xrange(len(ee_position))])
dist = np.linalg.norm(np.array(self.goal_position - ee_position))
if dist < self.goal_radius:
dist = 0.0
ee_position_distances.append(dist)
n, min_max, mean_distance_to_goal, var, skew, kurt = scipy.stats.describe(np.array(ee_position_distances))
self.serializer.write_line("log.log", tmp_dir, "################################# \n")
self.serializer.write_line("log.log",
tmp_dir,
"inc_covariance: " + str(self.inc_covariance) + "\n")
if self.inc_covariance == "process":
self.serializer.write_line("log.log",
tmp_dir,
"Process covariance: " + str(m_covs[j]) + " \n")
self.serializer.write_line("log.log",
tmp_dir,
"Observation covariance: " + str(self.min_observation_covariance) + " \n")
elif self.inc_covariance == "observation":
self.serializer.write_line("log.log",
tmp_dir,
"Process covariance: " + str(self.min_process_covariance) + " \n")
self.serializer.write_line("log.log",
tmp_dir,
"Observation covariance: " + str(m_covs[j]) + " \n")
num_steps /= self.num_simulation_runs
self.serializer.write_line("log.log", tmp_dir, "Mean number of steps: " + str(num_steps) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Objective value of best path: " + str(objective) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Mean num collisions per run: " + str(float(num_collisions) / float(self.num_simulation_runs)) + " \n")
print "collision_prob " + str(collided_num / float(self.num_simulation_runs))
print "total num collisions " + str(num_collisions)
print "mean num collisions " + str(float(num_collisions) / float(self.num_simulation_runs))
self.serializer.write_line("log.log", tmp_dir, "Length best path: " + str(len(xs)) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Index of best path: " + str(path_index) + " \n")
self.serializer.write_line("log.log",
tmp_dir,
"Average distance to goal area: " + str(mean_distance_to_goal) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Num successes: " + str(successes) + " \n")
print "succ " + str((100.0 / self.num_simulation_runs) * successes)
self.serializer.write_line("log.log", tmp_dir, "Percentage of successful runs: " + str((100.0 / self.num_simulation_runs) * successes) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Mean planning time: " + str(mean_planning_time) + " \n")
n, min_max, mean, var, skew, kurt = scipy.stats.describe(np.array(rewards_cov))
print "mean_rewards " + str(mean)
#plt.plot_histogram_from_data(rewards_cov)
#sleep
self.serializer.write_line("log.log", tmp_dir, "Mean rewards: " + str(mean) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Reward variance: " + str(var) + " \n")
self.serializer.write_line("log.log",
tmp_dir,
"Reward standard deviation: " + str(np.sqrt(var)) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Seed: " + str(self.seed) + " \n")
cmd = "mv " + tmp_dir + "/log.log " + dir + "/log_lqg_" + str(m_covs[j]) + ".log"
os.system(cmd)
if self.plot_paths:
self.serializer.save_paths(best_paths, 'best_paths.yaml', True, path=dir)
#self.serializer.save_stats(stats, path=dir)
cmd = "cp " + self.abs_path + "/config_lqg.yaml " + dir
os.system(cmd)
if not os.path.exists(dir + "/environment"):
os.makedirs(dir + "/environment")
cmd = "cp " + self.abs_path + "/" + str(self.environment_file) + " " + str(dir) + "/environment"
os.system(cmd)
if not os.path.exists(dir + "/model"):
os.makedirs(dir + "/model")
cmd = "cp " + self.abs_path + "/" + self.robot_file + " " + dir + "/model"
os.system(cmd)
print "LQG: Time to generate paths: " + str(time_to_generate_paths) + " seconds"
print "Done"
def calc_covariance_value(self, robot, error, covariance_matrix, covariance_type='process'):
active_joints = v_string()
robot.getActiveJoints(active_joints)
if covariance_type == 'process':
if self.dynamic_problem:
torque_limits = v_double()
robot.getJointTorqueLimits(active_joints, torque_limits)
torque_limits = [torque_limits[i] for i in xrange(len(torque_limits))]
for i in xrange(len(torque_limits)):
torque_range = 2.0 * torque_limits[i]
covariance_matrix[i, i] = np.square((torque_range / 100.0) * error)
else:
for i in xrange(self.robot_dof):
covariance_matrix[i, i] = np.square((self.max_velocity / 100.0) * error)
else:
lower_position_limits = v_double()
upper_position_limits = v_double()
velocity_limits = v_double()
robot.getJointLowerPositionLimits(active_joints, lower_position_limits)
robot.getJointUpperPositionLimits(active_joints, upper_position_limits)
robot.getJointVelocityLimits(active_joints, velocity_limits)
for i in xrange(self.robot_dof):
position_range = upper_position_limits[i] - lower_position_limits[i]
covariance_matrix[i, i] = np.square((position_range / 100.0) * error)
velocity_range = 2.0 * velocity_limits[i]
covariance_matrix[i + self.robot_dof, i + self.robot_dof] = np.square((velocity_range / 100.0) * error)
return covariance_matrix
def init_robot(self, urdf_model_file):
self.robot = Robot(self.abs_path + "/" + urdf_model_file)
self.robot.enforceConstraints(self.enforce_constraints)
self.robot.setGravityConstant(self.gravity_constant)
self.robot.setAccelerationLimit(self.acceleration_limit)
""" Setup operations """
self.robot_dof = self.robot.getDOF()
if len(self.start_state) != 2 * self.robot_dof:
logging.error("LQG: Start state dimension doesn't fit to the robot state space dimension")
return False
return True
"""
Analyzing functions (will be removed later)
=====================================================================
"""
def sample_control_error(self, M):
mu = np.array([0.0 for i in xrange(2 * self.robot_dof)])
return np.random.multivariate_normal(mu, M)
def show_state_distribution(self, model_file, env_file):
self.robot.setupViewer(model_file, env_file)
M = 30.0 * np.identity(2 * self.robot_dof)
active_joints = v_string()
self.robot.getActiveJoints(active_joints)
self.robot_dof = len(active_joints)
#x = [0.0, -np.pi / 2.0, 0.0, 0.0, 0.0, 0.0]
states = []
for z in xrange(2000):
u = [70.0, 70.0, 70.0, 0.0, 0.0, 0.0]
current_state = v_double()
current_state[:] = x
control = v_double()
control[:] = u
control_error = v_double()
ce = self.sample_control_error(M)
#ce = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
control_error[:] = ce
x = None
for k in xrange(1):
result = v_double()
self.robot.propagate(current_state,
control,
control_error,
self.simulation_step_size,
self.delta_t,
result)
x = [result[i] for i in xrange(len(result))]
current_state[:] = x
print x
states.append(np.array(x[3:6]))
x = [0.0, -np.pi/ 2.0, 0.0, 0.0, 0.0, 0.0]
cjvals = v_double()
cjvels = v_double()
cjvals_arr = [x[i] for i in xrange(len(x) / 2)]
cjvels_arr = [x[i] for i in xrange(len(x) / 2, len(x))]
cjvals[:] = cjvals_arr
cjvels[:] = cjvels_arr
particle_joint_values = v2_double()
particle_joint_colors = v2_double()
self.robot.updateViewerValues(cjvals,
cjvels,
particle_joint_values,
particle_joint_values)
from plot import plot_3d_points
mins = []
maxs = []
x_min = min([states[i][0] for i in xrange(len(states))])
x_max = max([states[i][0] for i in xrange(len(states))])
y_min = min([states[i][1] for i in xrange(len(states))])
y_max = max([states[i][1] for i in xrange(len(states))])
z_min = min([states[i][2] for i in xrange(len(states))])
z_max = max([states[i][2] for i in xrange(len(states))])
scale = [-0.2, 0.2]
plot_3d_points(np.array(states),
x_scale = [x_min, x_max],
y_scale = [y_min, y_max],
z_scale= [z_min, z_max])
sleep
def check_continuous_collide(self, model_file, env_file):
self.robot.setViewerBackgroundColor(0.6, 0.8, 0.6)
self.robot.setViewerSize(1280, 768)
self.robot.setupViewer(model_file, env_file)
x1 = [0.0115186,
0.1404386,
-0.17235397,
-0.22907283,
0.88922755,
1.35271925,
-3.38231963,
-1.51419397]
x2 = [0.04028825,
0.18442375,
-0.25677913,
-0.33145854,
0.81281939,
1.51040733,
-2.37524124,
-3.43838968]
#x1 = [np.pi - np.pi / 4, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
#x1 = [np.pi / 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
#x2 = [np.pi / 2.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
coll_t = 0.0
k = 0
while True:
k += 1
cjvals = v_double()
cjvels = v_double()
cjvals_arr = [x1[i] for i in xrange(len(x1) / 2)]
cjvels_arr = [x1[i] for i in xrange(len(x1) / 2, len(x1))]
cjvals[:] = cjvals_arr
cjvels[:] = cjvels_arr
particle_joint_values = v2_double()
ja_start = v_double()
ja_goal = v_double()
ja_start[:] = [x1[i] for i in xrange(len(x1) / 2)]
ja_goal[:] = [x2[i] for i in xrange(len(x2) / 2)]
collision_objects_start = self.robot.createRobotCollisionObjects(ja_start)
collision_objects_goal = self.robot.createRobotCollisionObjects(ja_goal)
in_collision_discrete = False
in_collision_continuous = False
t0 = time.time()
breaking = False
for o in self.obstacles:
#in_collision_discrete_start = o.inCollisionDiscrete(collision_objects_start)
#in_collision_discrete_goal = o.inCollisionDiscrete(collision_objects_goal)
for i in xrange(len(collision_objects_start)):
in_collision_continuous = o.inCollisionContinuous([collision_objects_start[i], collision_objects_goal[i]])
if in_collision_continuous:
break
if in_collision_continuous:
break
'''if in_collision_discrete_start:
print "start collides discrete"
in_collision_discrete = True
break
if in_collision_discrete_goal:
print "goal collides discrete"
in_collision_discrete = True
break
if in_collision_continuous:
print "collides continuous"
break
print "in collision discrete " + str(in_collision_discrete)
print "in collision continuous " + str(in_collision_continuous)'''
print "in collision continuous " + str(in_collision_continuous)
coll_t += time.time() - t0
if k == 10000:
print coll_t / k
sleep
self.robot.updateViewerValues(cjvals,
cjvels,
particle_joint_values,
particle_joint_values)
#time.sleep(10)
time.sleep(0.03)
def test_sensor(self, model_file, env_file, sensor_file):
show_viewer = True
if show_viewer:
self.robot.setViewerBackgroundColor(0.6, 0.8, 0.6)
self.robot.setViewerSize(1280, 768)
self.robot.setupViewer(model_file, env_file)
#self.robot.addSensor(sensor_file)
x = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
u = [0.0, 0.0, 0.0]
u_err = [0.0, 0.0, 0.0]
current_state = v_double()
control = v_double()
control[:] = u
control_error = v_double()
control_error[:] = u_err
result = v_double()
angles = v_double()
self.robot.setSensorTransform(current_state)
while True:
current_state[:] = x
self.robot.propagate(current_state,
control,
control_error,
self.simulation_step_size,
0.03,
result)
angles[:] = [result[i] for i in xrange(len(result) / 2)]
self.robot.setSensorTransform(angles)
x = np.array([result[i] for i in xrange(len(result))])
cjvals = v_double()
cjvels = v_double()
cjvals_arr = [x[i] for i in xrange(len(x) / 2)]
cjvels_arr = [x[i] for i in xrange(len(x) / 2, len(x))]
cjvals[:] = cjvals_arr
cjvels[:] = cjvels_arr
particle_joint_values = v2_double()
if show_viewer:
self.robot.updateViewerValues(cjvals,
cjvels,
particle_joint_values,
particle_joint_values)
time.sleep(0.03)
print "hello"
def run_viewer(self, model_file, env_file):
show_viewer = True
rot = v_double()
trans = v_double()
rot[:] = [-1.0, 0.0, 0.0, 0.0]
trans[:] = [0.0, 0.0, 3.0]
if show_viewer:
self.robot.setViewerBackgroundColor(0.6, 0.8, 0.6)
self.robot.setViewerSize(1280, 768)
self.robot.setupViewer(model_file, env_file)
fx = 0.0
fy = 0.0
fz = 0.0
f_roll = 0.0
f_pitch = 0.0
f_yaw = 0.0
x = [0.0 for i in xrange(2 * self.robot_dof)]
''''x = [0.0, 0.0, 0.0, 0.0,
0.0,
0.0,
0.0,
0.0]'''
#x = [1.78349, 0.868347, -0.927761]
x[1] = 0.6
x_true = [0.0 for i in xrange(2 * self.robot_dof)]
integration_times = []
collision_check_times1 = []
collision_check_times2 = []
current_state = v_double()
y = 0
while True:
u_in = [0.0 for i in xrange(self.robot_dof)]
u_in[0] = 150.0
current_state[:] = x
control = v_double()
control[:] = u_in
control_error = v_double()
ce = [0.0 for i in xrange(self.robot_dof)]
control_error[:] = ce
result = v_double()
###########################
ee_jacobian = v2_double()
self.robot.getEndEffectorJacobian(current_state, ee_jacobian);
###########################
t0 = time.time()
self.robot.propagate(current_state,
control,
control_error,
self.simulation_step_size,
0.03,
result)
#print result[11]
t = time.time() - t0
integration_times.append(t)
if y == 1000:
t_sum = sum(integration_times)
t_mean = t_sum / len(integration_times)
print "mean integration times: " + str(t_mean)
t_sum = sum(collision_check_times1)
t_mean = t_sum / len(collision_check_times1)
print "mean collision check times old " + str(t_mean)
t_mean = sum(collision_check_times2) / len(collision_check_times2)
print "mean collision check times new " + str(t_mean)
sleep
ja_start = v_double()
ja_start[:] = [current_state[i] for i in xrange(len(current_state) / 2)]
collision_objects_start = self.robot.createRobotCollisionObjects(ja_start)
joint_angles = v_double()
joint_angles[:] = [result[i] for i in xrange(len(result) / 2)]
collision_objects_goal = self.robot.createRobotCollisionObjects(joint_angles)
#print "ee_velocity " + str([ee_velocity[i] for i in xrange(len(ee_velocity))])
t_coll_check1 = time.time()
t2 = time.time() - t_coll_check1
collision_check_times2.append(t2)
'''
Get the end effector position
'''
#ee_position = v_double()
#self.robot.getEndEffectorPosition(joint_angles, ee_position)
#ee_collision_objects = self.robot.createEndEffectorCollisionObject(joint_angles)
in_collision = False
t_c = time.time()
for o in self.obstacles:
if o.inCollisionDiscrete(collision_objects_goal):
in_collision = True
print "IN COLLISION"
break
'''for i in xrange(len(collision_objects_start)):
if o.inCollisionContinuous([collision_objects_start[i], collision_objects_goal[i]]):
in_collision = True
break'''
if in_collision:
break
t = time.time() - t_c
collision_check_times1.append(t)
x = np.array([result[i] for i in xrange(len(result))])
cjvals = v_double()
cjvels = v_double()
cjvals_arr = [x[i] for i in xrange(len(x) / 2)]
cjvels_arr = [x[i] for i in xrange(len(x) / 2, len(x))]
cjvals[:] = cjvals_arr
cjvels[:] = cjvels_arr
particle_joint_values = v2_double()
if show_viewer:
self.robot.updateViewerValues(cjvals,
cjvels,
particle_joint_values,
particle_joint_values)
#time.sleep(0.03)
time.sleep(0.03)
y += 1
print y
"""
================================================================
"""
def setup_scene(self,
environment_file,
robot):
""" Load the obstacles """
self.obstacles = self.utils.loadObstaclesXML(self.abs_path + "/" + environment_file)
""" Load the goal area """
goal_area = v_double()
self.utils.loadGoalArea(self.abs_path + "/" + environment_file, goal_area)
if len(goal_area) == 0:
print "ERROR: Your environment file doesn't define a goal area"
return False
self.goal_position = [goal_area[i] for i in xrange(0, 3)]
self.goal_radius = goal_area[3]
return True
def init_serializer(self):
self.serializer = Serializer()
self.serializer.create_temp_dir(self.abs_path, "lqg")
def clear_stats(self, dir):
if os.path.isdir(dir):
cmd = "rm -rf " + dir + "/*"
os.system(cmd)
else:
os.makedirs(dir)
def get_average_distance_to_goal_area(self, goal_position, goal_radius, cartesian_coords):
avg_dist = 0.0
goal_pos = np.array(goal_position)
for i in xrange(len(cartesian_coords)):
cart = np.array(cartesian_coords[i])
dist = np.linalg.norm(goal_pos - cart)
if dist < goal_radius:
dist = 0.0
avg_dist += dist
if avg_dist == 0.0:
return avg_dist
return np.asscalar(avg_dist) / len(cartesian_coords)
def problem_setup(self, delta_t, num_links):
A = np.identity(num_links * 2)
H = np.identity(num_links * 2)
W = np.identity(num_links * 2)
C = self.path_deviation_cost * np.identity(num_links * 2)
'''B = delta_t * np.identity(num_links * 2)
V = np.identity(num_links * 2)
D = self.control_deviation_cost * np.identity(num_links * 2)
M_base = np.identity(2 * self.robot_dof)
N_base = np.identity(2 * self.robot_dof)'''
B = delta_t * np.vstack((np.identity(num_links),
np.zeros((num_links, num_links))))
V = np.vstack((np.identity(num_links),
np.zeros((num_links, num_links))))
D = self.control_deviation_cost * np.identity(num_links)
M_base = np.identity(self.robot_dof)
N_base = np.identity(2 * self.robot_dof)
return A, H, B, V, W, C, D, M_base, N_base
def get_avg_path_length(self, paths):
avg_length = 0.0
for path in paths:
avg_length += len(path[0])
return avg_length / len(paths)
def set_params(self, config):
self.num_paths = config['num_generated_paths']
self.use_linear_path = config['use_linear_path']
self.max_velocity = config['max_velocity']
self.delta_t = 1.0 / config['control_rate']
self.start_state = config['start_state']
self.num_simulation_runs = config['num_simulation_runs']
self.num_bins = config['num_bins']
self.min_process_covariance = config['min_process_covariance']
self.max_process_covariance = config['max_process_covariance']
self.covariance_steps = config['covariance_steps']
self.min_observation_covariance = config['min_observation_covariance']
self.max_observation_covariance = config['max_observation_covariance']
self.discount_factor = config['discount_factor']
self.illegal_move_penalty = config['illegal_move_penalty']
self.step_penalty = config['step_penalty']
self.exit_reward = config['exit_reward']
self.stop_when_terminal = config['stop_when_terminal']
self.enforce_constraints = config['enforce_constraints']
self.enforce_control_constraints = config['enforce_control_constraints']
self.sample_size = config['sample_size']
self.plot_paths = config['plot_paths']
self.planning_algortihm = config['planning_algorithm']
self.dynamic_problem = config['dynamic_problem']
self.simulation_step_size = config['simulation_step_size']
self.path_timeout = config['path_timeout']
self.continuous_collision = config['continuous_collision_check']
self.show_viewer_evaluation = config['show_viewer_evaluation']
self.show_viewer_simulation = config['show_viewer_simulation']
self.path_deviation_cost = config['path_deviation_cost']
self.control_deviation_cost = config['control_deviation_cost']
self.num_control_samples = config['num_control_samples']
self.min_control_duration = config['min_control_duration']
self.max_control_duration = config['max_control_duration']
self.inc_covariance = config['inc_covariance']
self.add_intermediate_states = config['add_intermediate_states']
self.gravity_constant = config['gravity']
self.num_generated_goal_states = config['num_generated_goal_states']
self.robot_file = config['robot_file']
self.environment_file = config['environment_file']
self.rrt_goal_bias = config['rrt_goal_bias']
self.control_sampler = config['control_sampler']
self.seed = config['seed']
self.num_cores = config['num_cores']
self.acceleration_limit = config['acceleration_limit']
class TestSerialize(unittest.TestCase):
@classmethod
def _clearMsgDefs(self):
# Clear all non-test messages
try:
while True:
messages.MESSAGES.pop()
except IndexError:
pass
@classmethod
def setUpClass(self):
self._clearMsgDefs()
messages.MESSAGES.append(__import__("test_pb2", globals(), locals(), [], -1))
self.blank_args = [1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7, 0, 8, 0, 9, 0, 10, 0, 11, 0, 12, 0, True, 0, "a", "b", "c", "d"]
self.neg_args = [-1.0, 0, -1.0, 0, -1, -1, -1, -1, 0, 0, 0, 0, -1, -1, -1, -1, 0, 0, 0, 0, -1, -1, -1, -1, 1, 1, "", "", "", ""]
self.illegal_len_args = [
[0, 1.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 1.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""]
]
self.too_long_args = [
[0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32768, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, -32769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, -32769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -32769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -32769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -32769, 0, 0, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -32769, 0, 0, "", "", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "aaa", "", ""],
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", "", "aaa"]
]
def setUp(self):
self.s = Serializer()
def test_id(self):
enc = self.s.serialize("Test", *self.blank_args)
(name, ) = struct.unpack_from('20s', enc)
self.assertTrue(name.startswith('test_pb2Test'))
def test_values(self):
enc = self.s.serialize("Test", *self.blank_args)
def f(obj):
self.assertTrue(obj.IsInitialized())
for (_, value) in obj.ListFields():
self.assertIn(value, self.blank_args)
self.s.add_handler("Test", f)
self.s.unserialize(enc)
def test_explicit_call(self):
enc = self.s.serialize("test.Test", *self.blank_args)
def f(obj):
self.assertTrue(obj.IsInitialized())
for (_, value) in obj.ListFields():
self.assertIn(value, self.blank_args)
self.s.add_handler("Test", f)
self.s.unserialize(enc)
def test_negative_values(self):
enc = self.s.serialize("Test", *self.neg_args)
def f(obj):
self.assertTrue(obj.IsInitialized())
for (_, value) in obj.ListFields():
self.assertIn(value, self.neg_args)
self.s.add_handler("Test", f)
self.s.unserialize(enc)
def test_illegal_length_option(self):
for args in self.illegal_len_args:
with self.assertRaises(FieldLengthUnsupportedException):
self.s.serialize("Test", *args)
def test_too_long_option(self):
for args in self.too_long_args:
with self.assertRaises(FieldTooLongException):
self.s.serialize("Test", *args)
def test_unknown_message(self):
with self.assertRaises(UnknownMessageException):
self.s.serialize("Unknown")
def test_unknown_message_explicit(self):
with self.assertRaises(UnknownMessageException):
self.s.serialize("test.Unknown")
def test_unknown_message_explicit2(self):
with self.assertRaises(UnknownMessageException):
self.s.serialize("t.Unknown")
def test_incomplete_definition_empty(self):
with self.assertRaises(FieldNotDefinedException):
self.s.serialize("Test")
def test_incomplete_definition_unnamed(self):
with self.assertRaises(FieldNotDefinedException):
self.s.serialize("Test", 0)
def test_incomplete_definition_named(self):
with self.assertRaises(FieldNotDefinedException):
self.s.serialize("Test", boolnolen = 0)
def test_incomplete_definition_hybrid(self):
with self.assertRaises(FieldNotDefinedException):
self.s.serialize("Test", 0, boolnolen = 0)
def test_wrong_type(self):
args = self.blank_args[:]
args[0] = ""
with self.assertRaises(FieldWrongTypeException):
self.s.serialize("Test", *args)
def test_wrong_type_value(self):
args = self.blank_args[:]
args[5] = 9999999999999999
with self.assertRaises(FieldWrongTypeException):
self.s.serialize("Test", *args)
def test_hash_name(self):
name = "test"
for i in range(1,9):
self.assertEqual(len(self.s._hash_name(name, i)),
i)
def test_reload_definitions(self):
class T(logging.Filter):
def filter(self, record):
self.called = True
return 0
filter = T()
filter.called = False
logging.getLogger().addFilter(filter)
self.assertIn("'test_pb2Test\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00'", self.s.messages)
self.s.load_definitions()
self.assertIn("'test_pb2Test\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00'", self.s.messages)
self.assertTrue(filter.called)
logging.getLogger().removeFilter(filter)
def test_wrong_type_value_check(self):
args = self.blank_args[:]
args[1] = 1.10
with self.assertRaises(FieldLengthUnsupportedException):
self.s.serialize("Test", *args)
def setUp(self):
self.s = Serializer()
def init_serializer(self):
self.serializer = Serializer()
self.serializer.create_temp_dir(self.abs_path, "mpc")
class MPC:
def __init__(self, plot):
self.abs_path = os.path.dirname(os.path.abspath(__file__))
""" Reading the config """
warnings.filterwarnings("ignore")
self.init_serializer()
config = self.serializer.read_config("config_mpc.yaml", path=self.abs_path)
self.set_params(config)
if self.seed < 0:
"""
Generate a random seed that will be stored
"""
self.seed = np.random.randint(0, 4294967295)
np.random.seed(self.seed)
logging_level = logging.WARN
if config['verbose']:
logging_level = logging.DEBUG
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging_level)
np.set_printoptions(precision=16)
dir = self.abs_path + "/stats/mpc"
tmp_dir = self.abs_path + "/tmp/mpc"
self.utils = Utils()
if not self.init_robot(self.robot_file):
logging.error("MPC: Couldn't initialize robot")
return
if not self.setup_scene(self.environment_file, self.robot):
return
#self.run_viewer(self.robot_file, self.environment_file)
self.clear_stats(dir)
logging.info("Start up simulator")
sim = Simulator()
path_evaluator = PathEvaluator()
path_planner = PathPlanningInterface()
logging.info("MPC: Generating goal states...")
goal_states = get_goal_states("mpc",
self.abs_path,
self.serializer,
self.obstacles,
self.robot,
self.max_velocity,
self.delta_t,
self.start_state,
self.goal_position,
self.goal_radius,
self.planning_algortihm,
self.path_timeout,
self.num_generated_goal_states,
self.continuous_collision,
self.environment_file,
self.num_cores)
if len(goal_states) == 0:
logging.error("MPC: Couldn't generate any goal states. Problem seems to be infeasible")
return
logging.info("MPC: Generated " + str(len(goal_states)) + " goal states")
sim.setup_reward_function(self.discount_factor, self.step_penalty, self.illegal_move_penalty, self.exit_reward)
path_planner.setup(self.robot,
self.obstacles,
self.max_velocity,
self.delta_t,
self.use_linear_path,
self.planning_algortihm,
self.path_timeout,
self.continuous_collision,
self.num_cores)
if self.dynamic_problem:
path_planner.setup_dynamic_problem(self.robot_file,
self.environment_file,
self.simulation_step_size,
self.num_control_samples,
self.min_control_duration,
self.max_control_duration,
self.add_intermediate_states,
self.rrt_goal_bias,
self.control_sampler)
A, H, B, V, W, C, D, M_base, N_base = self.problem_setup(self.delta_t, self.robot_dof)
time_to_generate_paths = 0.0
if check_positive_definite([C, D]):
m_covs = None
if self.inc_covariance == "process":
m_covs = np.linspace(self.min_process_covariance,
self.max_process_covariance,
self.covariance_steps)
elif self.inc_covariance == "observation":
m_covs = np.linspace(self.min_observation_covariance,
self.max_observation_covariance,
self.covariance_steps)
for j in xrange(len(m_covs)):
M = None
N = None
if self.inc_covariance == "process":
""" The process noise covariance matrix """
M = self.calc_covariance_value(self.robot, m_covs[j], M_base)
#M = m_covs[j] * M_base
""" The observation error covariance matrix """
N = self.calc_covariance_value(self.robot,
self.min_observation_covariance,
N_base,
covariance_type='observation')
elif self.inc_covariance == "observation":
M = self.calc_covariance_value(self.robot,
self.min_process_covariance,
M_base)
N = self.calc_covariance_value(self.robot,
m_covs[j],
N_base,
covariance_type='observation')
P_t = np.array([[0.0 for k in xrange(2 * self.robot_dof)] for l in xrange(2 * self.robot_dof)])
path_planner.setup_path_evaluator(A, B, C, D, H, M, N, V, W,
self.robot,
self.sample_size,
self.obstacles,
self.goal_position,
self.goal_radius,
self.robot_file,
self.environment_file)
sim.setup_problem(A, B, C, D, H, V, W, M, N,
self.robot,
self.enforce_control_constraints,
self.obstacles,
self.goal_position,
self.goal_radius,
self.max_velocity,
self.show_viewer_simulation,
self.robot_file,
self.environment_file)
sim.set_stop_when_colliding(self.replan_when_colliding)
if self.dynamic_problem:
path_evaluator.setup_dynamic_problem()
sim.setup_dynamic_problem(self.simulation_step_size)
path_planner.setup_reward_function(self.step_penalty, self.exit_reward, self.illegal_move_penalty, self.discount_factor)
mean_number_planning_steps = 0.0
number_of_steps = 0.0
mean_planning_time = 0.0
num_generated_paths_run = 0.0
successful_runs = 0
num_collisions = 0.0
final_states= []
rewards_cov = []
for k in xrange(self.num_simulation_runs):
print "MPC: Run " + str(k + 1)
self.serializer.write_line("log.log", tmp_dir, "RUN #" + str(k + 1) + " \n")
current_step = 0
x_true = self.start_state
x_estimate = self.start_state
x_tilde_linear = np.array([0.0 for i in xrange(2 * self.robot_dof)])
P_t = np.array([[0.0 for i in xrange(2 * self.robot_dof)] for i in xrange(2 * self.robot_dof)])
deviation_covariance = np.array([[0.0 for i in xrange(2 * self.robot_dof)] for i in xrange(2 * self.robot_dof)])
estimated_deviation_covariance = np.array([[0.0 for i in xrange(2 * self.robot_dof)] for i in xrange(2 * self.robot_dof)])
total_reward = 0.0
terminal = False
while current_step < self.max_num_steps and not terminal:
path_planner.set_start_and_goal(x_estimate, goal_states, self.goal_position, self.goal_radius)
t0 = time.time()
(xs,
us,
zs,
control_durations,
num_generated_paths,
best_val,
state_covariances,
deviation_covariances,
estimated_deviation_covariances,
mean_gen_times,
mean_eval_times,
total_gen_times,
total_eval_times) = path_planner.plan_and_evaluate_paths(self.num_paths,
0,
current_step,
self.evaluation_horizon,
P_t,
deviation_covariance,
estimated_deviation_covariance,
self.timeout)
mean_planning_time += time.time() - t0
mean_number_planning_steps += 1.0
num_generated_paths_run += num_generated_paths
if len(xs) == 0:
logging.error("MPC: Couldn't find any paths from start state" +
str(x_estimate) +
" to goal states")
total_reward = np.array([-self.illegal_move_penalty])[0]
current_step += 1
break
x_tilde = np.array([0.0 for i in xrange(2 * self.robot_dof)])
n_steps = self.num_execution_steps
if n_steps > len(xs) - 1:
n_steps = len(xs) - 1
if current_step + n_steps > self.max_num_steps:
n_steps = self.max_num_steps - current_step
(x_true,
x_tilde,
x_tilde_linear,
x_estimate,
P_t,
current_step,
total_reward,
terminal,
estimated_s,
estimated_c,
history_entries) = sim.simulate_n_steps(xs, us, zs,
control_durations,
x_true,
x_tilde,
x_tilde_linear,
x_estimate,
P_t,
total_reward,
current_step,
n_steps,
0.0,
0.0,
max_num_steps=self.max_num_steps)
#print "len(hist) " + str(len(history_entries))
#print "len(deviation_covariances) " + str(len(deviation_covariances))
#print "len(estimated_deviation_covariances) " + str(len(estimated_deviation_covariances))
deviation_covariance = deviation_covariances[len(history_entries) - 1]
estimated_deviation_covariance = estimated_deviation_covariances[len(history_entries) - 1]
if (current_step == self.max_num_steps) or terminal:
final_states.append(history_entries[-1].x_true)
print "len " + str(len(history_entries))
print "t " + str(history_entries[-1].t)
if terminal:
successful_runs += 1
history_entries[0].set_replanning(True)
for l in xrange(len(history_entries)):
history_entries[l].set_estimated_covariance(state_covariances[l])
history_entries[l].serialize(tmp_dir, "log.log")
if history_entries[l].collided:
num_collisions += 1
collided = True
#logging.warn("MPC: Execution finished. True state is " + str(x_true))
#logging.warn("MPC: Estimated state is " + str(x_estimate))
logging.warn("MPC: Executed " + str(current_step) + " steps")
logging.warn("MPC: terminal " + str(terminal))
if terminal:
print "MPC: Final state: " + str(x_true)
rewards_cov.append(total_reward)
self.serializer.write_line("log.log",
tmp_dir,
"Reward: " + str(total_reward) + " \n")
self.serializer.write_line("log.log",
tmp_dir,
"\n")
number_of_steps += current_step
mean_planning_time_per_step = mean_planning_time / mean_number_planning_steps
mean_planning_time /= self.num_simulation_runs
""" Calculate the distance to goal area
"""
ee_position_distances = []
for state in final_states:
joint_angles = v_double()
joint_angles[:] = [state[y] for y in xrange(len(state) / 2)]
ee_position = v_double()
self.robot.getEndEffectorPosition(joint_angles, ee_position)
ee_position = np.array([ee_position[y] for y in xrange(len(ee_position))])
dist = np.linalg.norm(np.array(self.goal_position - ee_position))
if dist < self.goal_radius:
dist = 0.0
ee_position_distances.append(dist)
logging.info("MPC: Done. total_reward is " + str(total_reward))
try:
n, min_max, mean_distance_to_goal, var, skew, kurt = scipy.stats.describe(np.array(ee_position_distances))
except:
print ee_position_distances
sleep
self.serializer.write_line("log.log", tmp_dir, "################################# \n")
self.serializer.write_line("log.log",
tmp_dir,
"inc_covariance: " + str(self.inc_covariance) + "\n")
if self.inc_covariance == "process":
self.serializer.write_line("log.log",
tmp_dir,
"Process covariance: " + str(m_covs[j]) + " \n")
self.serializer.write_line("log.log",
tmp_dir,
"Observation covariance: " + str(self.min_observation_covariance) + " \n")
elif self.inc_covariance == "observation":
self.serializer.write_line("log.log",
tmp_dir,
"Process covariance: " + str(self.min_process_covariance) + " \n")
self.serializer.write_line("log.log",
tmp_dir,
"Observation covariance: " + str(m_covs[j]) + " \n")
number_of_steps /= self.num_simulation_runs
self.serializer.write_line("log.log", tmp_dir, "Mean number of steps: " + str(number_of_steps) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Mean num collisions per run: " + str(float(num_collisions) / float(self.num_simulation_runs)) + " \n")
self.serializer.write_line("log.log",
tmp_dir,
"Average distance to goal area: " + str(mean_distance_to_goal) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Num successes: " + str(successful_runs) + " \n")
print "succ " + str((100.0 / self.num_simulation_runs) * successful_runs)
self.serializer.write_line("log.log", tmp_dir, "Percentage of successful runs: " + str((100.0 / self.num_simulation_runs) * successful_runs) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Mean planning time per run: " + str(mean_planning_time) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Mean planning time per planning step: " + str(mean_planning_time_per_step) + " \n")
n, min_max, mean, var, skew, kurt = scipy.stats.describe(np.array(rewards_cov))
print "mean_rewards " + str(mean)
#plt.plot_histogram_from_data(rewards_cov)
#sleep
self.serializer.write_line("log.log", tmp_dir, "Mean rewards: " + str(mean) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Reward variance: " + str(var) + " \n")
self.serializer.write_line("log.log",
tmp_dir,
"Reward standard deviation: " + str(np.sqrt(var)) + " \n")
self.serializer.write_line("log.log", tmp_dir, "Seed: " + str(self.seed) + " \n")
cmd = "mv " + tmp_dir + "/log.log " + dir + "/log_mpc_" + str(m_covs[j]) + ".log"
os.system(cmd)
cmd = "cp " + self.abs_path + "/config_mpc.yaml " + dir
os.system(cmd)
if not os.path.exists(dir + "/environment"):
os.makedirs(dir + "/environment")
cmd = "cp " + self.abs_path + "/" + str(self.environment_file) + " " + str(dir) + "/environment"
os.system(cmd)
if not os.path.exists(dir + "/model"):
os.makedirs(dir + "/model")
cmd = "cp " + self.abs_path + "/" + self.robot_file + " " + dir + "/model"
os.system(cmd)
print "Done."
def init_robot(self, urdf_model_file):
self.robot = Robot(self.abs_path + "/" + urdf_model_file)
self.robot.enforceConstraints(self.enforce_constraints)
self.robot.setGravityConstant(self.gravity_constant)
self.robot.setAccelerationLimit(self.acceleration_limit)
""" Setup operations """
self.robot_dof = self.robot.getDOF()
if len(self.start_state) != 2 * self.robot_dof:
logging.error("MPC: Start state dimension doesn't fit to the robot state space dimension")
return False
return True
def run_viewer(self, model_file, env_file):
show_viewer = True
rot = v_double()
trans = v_double()
rot[:] = [-1.0, 0.0, 0.0, 0.0]
trans[:] = [0.0, 0.0, 3.0]
if show_viewer:
self.robot.setViewerBackgroundColor(0.6, 0.8, 0.6)
self.robot.setViewerSize(1280, 768)
self.robot.setupViewer(model_file, env_file)
fx = 0.0
fy = 0.0
fz = 0.0
f_roll = 0.0
f_pitch = 0.0
f_yaw = 0.0
x = [0.0 for i in xrange(2 * self.robot_dof)]
x = [0.8110760662014179,
-0.2416850600576381,
1.5922944779127346,
5.1412306972285471,
-10.0,
2.5101115097604483]
integration_times = []
collision_check_times1 = []
collision_check_times2 = []
y = 0
while True:
#u_in = [3.0, 1.5, 0.0, 0.0, 0.0, 0.0]
u_in = [0.0 for i in xrange(self.robot_dof)]
#u_in[0] = 150.0
#u_in[1] = 70.0
current_state = v_double()
current_state[:] = x
control = v_double()
control[:] = u_in
control_error = v_double()
ce = [0.0 for i in xrange(self.robot_dof)]
control_error[:] = ce
result = v_double()
t0 = time.time()
self.robot.propagate(current_state,
control,
control_error,
self.simulation_step_size,
0.03,
result)
t = time.time() - t0
integration_times.append(t)
if y == 10000:
t_sum = sum(integration_times)
t_mean = t_sum / len(integration_times)
print "mean integration times: " + str(t_mean)
t_sum = sum(collision_check_times1)
t_mean = t_sum / len(collision_check_times1)
print "mean collision check times old " + str(t_mean)
t_mean = sum(collision_check_times2) / len(collision_check_times2)
print "mean collision check times new " + str(t_mean)
sleep
ja_start = v_double()
ja_start[:] = [current_state[i] for i in xrange(len(current_state) / 2)]
collision_objects_start = self.robot.createRobotCollisionObjects(ja_start)
joint_angles = v_double()
joint_angles[:] = [result[i] for i in xrange(len(result) / 2)]
collision_objects_goal = self.robot.createRobotCollisionObjects(joint_angles)
#print "ee_velocity " + str([ee_velocity[i] for i in xrange(len(ee_velocity))])
t_coll_check1 = time.time()
t2 = time.time() - t_coll_check1
collision_check_times2.append(t2)
'''
Get the end effector position
'''
#ee_position = v_double()
#self.robot.getEndEffectorPosition(joint_angles, ee_position)
#ee_collision_objects = self.robot.createEndEffectorCollisionObject(joint_angles)
in_collision = False
t_c = time.time()
for o in self.obstacles:
if o.inCollisionDiscrete(collision_objects_goal):
in_collision = True
break
'''for i in xrange(len(collision_objects_start)):
if o.inCollisionContinuous([collision_objects_start[i], collision_objects_goal[i]]):
in_collision = True
break'''
if in_collision:
print "COLLISION"
t = time.time() - t_c
collision_check_times1.append(t)
x = np.array([result[i] for i in xrange(len(result))])
cjvals = v_double()
cjvels = v_double()
cjvals_arr = [x[i] for i in xrange(len(x) / 2)]
cjvels_arr = [x[i] for i in xrange(len(x) / 2, len(x))]
cjvals[:] = cjvals_arr
cjvels[:] = cjvels_arr
particle_joint_values = v2_double()
if show_viewer:
self.robot.updateViewerValues(cjvals,
cjvels,
particle_joint_values,
particle_joint_values)
time.sleep(0.03)
y += 1
print y
def setup_scene(self,
environment_file,
robot):
""" Load the obstacles """
self.obstacles = self.utils.loadObstaclesXML(self.abs_path + "/" + environment_file)
""" Load the goal area """
goal_area = v_double()
self.utils.loadGoalArea(self.abs_path + "/" + environment_file, goal_area)
if len(goal_area) == 0:
print "ERROR: Your environment file doesn't define a goal area"
return False
self.goal_position = [goal_area[i] for i in xrange(0, 3)]
self.goal_radius = goal_area[3]
return True
def init_serializer(self):
self.serializer = Serializer()
self.serializer.create_temp_dir(self.abs_path, "mpc")
def clear_stats(self, dir):
if os.path.isdir(dir):
cmd = "rm -rf " + dir + "/*"
os.system(cmd)
else:
os.makedirs(dir)
def get_average_distance_to_goal_area(self, goal_position, goal_radius, cartesian_coords):
avg_dist = 0.0
goal_pos = np.array(goal_position)
for i in xrange(len(cartesian_coords)):
cart = np.array(cartesian_coords[i])
dist = np.linalg.norm(goal_pos - cart)
if dist < goal_radius:
dist = 0.0
avg_dist += dist
if avg_dist == 0.0:
return avg_dist
return np.asscalar(avg_dist) / len(cartesian_coords)
def calc_covariance_value(self, robot, error, covariance_matrix, covariance_type='process'):
active_joints = v_string()
robot.getActiveJoints(active_joints)
if covariance_type == 'process':
if self.dynamic_problem:
torque_limits = v_double()
robot.getJointTorqueLimits(active_joints, torque_limits)
torque_limits = [torque_limits[i] for i in xrange(len(torque_limits))]
for i in xrange(len(torque_limits)):
torque_range = 2.0 * torque_limits[i]
covariance_matrix[i, i] = np.square((torque_range / 100.0) * error)
else:
for i in xrange(self.robot_dof):
covariance_matrix[i, i] = np.square((self.max_velocity / 100.0) * error)
else:
lower_position_limits = v_double()
upper_position_limits = v_double()
velocity_limits = v_double()
robot.getJointLowerPositionLimits(active_joints, lower_position_limits)
robot.getJointUpperPositionLimits(active_joints, upper_position_limits)
robot.getJointVelocityLimits(active_joints, velocity_limits)
for i in xrange(self.robot_dof):
position_range = upper_position_limits[i] - lower_position_limits[i]
covariance_matrix[i, i] = np.square((position_range / 100.0) * error)
velocity_range = 2.0 * velocity_limits[i]
covariance_matrix[i + self.robot_dof, i + self.robot_dof] = np.square((velocity_range / 100.0) * error)
return covariance_matrix
def problem_setup(self, delta_t, num_links):
A = np.identity(num_links * 2)
H = np.identity(num_links * 2)
W = np.identity(num_links * 2)
C = self.path_deviation_cost * np.identity(num_links * 2)
'''B = delta_t * np.identity(num_links * 2)
V = np.identity(num_links * 2)
D = self.control_deviation_cost * np.identity(num_links * 2)
M_base = np.identity(2 * self.robot_dof)
N_base = np.identity(2 * self.robot_dof)'''
B = delta_t * np.vstack((np.identity(num_links),
np.zeros((num_links, num_links))))
V = np.vstack((np.identity(num_links),
np.zeros((num_links, num_links))))
D = self.control_deviation_cost * np.identity(num_links)
M_base = np.identity(self.robot_dof)
N_base = np.identity(2 * self.robot_dof)
return A, H, B, V, W, C, D, M_base, N_base
def set_params(self, config):
self.num_paths = config['num_generated_paths']
self.use_linear_path = config['use_linear_path']
self.max_velocity = config['max_velocity']
self.delta_t = 1.0 / config['control_rate']
self.start_state = config['start_state']
self.num_simulation_runs = config['num_simulation_runs']
self.num_bins = config['num_bins']
self.min_process_covariance = config['min_process_covariance']
self.max_process_covariance = config['max_process_covariance']
self.covariance_steps = config['covariance_steps']
self.min_observation_covariance = config['min_observation_covariance']
self.max_observation_covariance = config['max_observation_covariance']
self.discount_factor = config['discount_factor']
self.illegal_move_penalty = config['illegal_move_penalty']
self.step_penalty = config['step_penalty']
self.exit_reward = config['exit_reward']
self.stop_when_terminal = config['stop_when_terminal']
self.enforce_constraints = config['enforce_constraints']
self.enforce_control_constraints = config['enforce_control_constraints']
self.sample_size = config['sample_size']
self.plot_paths = config['plot_paths']
self.planning_algortihm = config['planning_algorithm']
self.dynamic_problem = config['dynamic_problem']
self.simulation_step_size = config['simulation_step_size']
self.path_timeout = config['path_timeout']
self.continuous_collision = config['continuous_collision_check']
self.show_viewer_evaluation = config['show_viewer_evaluation']
self.show_viewer_simulation = config['show_viewer_simulation']
self.path_deviation_cost = config['path_deviation_cost']
self.control_deviation_cost = config['control_deviation_cost']
self.num_control_samples = config['num_control_samples']
self.min_control_duration = config['min_control_duration']
self.max_control_duration = config['max_control_duration']
self.inc_covariance = config['inc_covariance']
self.add_intermediate_states = config['add_intermediate_states']
self.gravity_constant = config['gravity']
self.num_generated_goal_states = config['num_generated_goal_states']
self.robot_file = config['robot_file']
self.environment_file = config['environment_file']
self.rrt_goal_bias = config['rrt_goal_bias']
self.control_sampler = config['control_sampler']
self.max_num_steps = config['max_num_steps']
self.evaluation_horizon = config['horizon']
self.timeout = config['timeout']
self.seed = config['seed']
self.num_cores = config['num_cores']
self.replan_when_colliding = config['replan_when_colliding']
self.acceleration_limit = config['acceleration_limit']
"""
The number of steps a path is being executed before a new path gets calculated
"""
self.num_execution_steps = config['num_execution_steps']
class ApiController:
def __init__(self, model, **kwargs):
self.model = model
self.fields = kwargs.get("fields") or self.model._meta.get_all_field_names()
self.name = kwargs.get("name")
self.parent_name = kwargs.get("parent_name", False)
if self.parent_name:
self.parent_id = self.parent_name + "_id"
else:
self.parent_id = ""
self.id = self.name + "_id"
self.set_name = self.name + "_set"
self.relations = kwargs.get("relations") or [
rel.get_accessor_name() for rel in model._meta.get_all_related_objects()
]
self.parent_model = kwargs.get("parent_model")
self.serialize = Serializer(self.fields)
@csrf_exempt
def nested_all(self, request, **kwargs):
if request.method == "POST":
return run_response(self.create, request, kwargs)
else:
return run_response(self.nested_list, request, kwargs)
@csrf_exempt
def all(self, request, **kwargs):
if request.method == "POST":
return run_response(self.create, request, kwargs)
else:
return run_response(self.list, request, kwargs)
@csrf_exempt
def all_or_authenticate(self, request, **kwargs):
if request.method == "POST":
return run_response(self.create_user, request, kwargs)
else:
return run_response(self.list, request, kwargs)
@csrf_exempt
def single(self, request, **kwargs):
if request.method == "POST" or request.method == "PATCH" or request.method == "PUT":
return run_response(self.update, request, kwargs)
elif request.method == "DELETE":
return run_response(self.destroy, request, kwargs)
else:
return run_response(self.show, request, kwargs)
def create_user(self, request, params):
attrs = self.serialize.to_dict(request.data)
if params.get(self.parent_id):
attrs[self.parent_id] = params[self.parent_id]
user = self.model(**attrs)
user.save()
token = jwt.encode({"user_id": user.id}, SECRET_KEY)
return {"success": True, "token": token, "user": self.serialize.model_to_dict(user)}
def create(self, request, params):
attrs = self.serialize.to_dict(request.data)
if params.get(self.parent_id):
attrs[self.parent_id] = params[self.parent_id]
record = self.model(**attrs)
record.save()
return self.serialize.model(record)
def update(self, request, params):
filtered = self.model.objects.filter(pk=params[self.id])
filtered.update(**self.serialize.to_dict(request.data))
return self.serialize.model(filtered[0])
def destroy(self, request, params):
record = self.model.objects.get(pk=params[self.id])
record.delete()
return {"status": "success", "message": "Record deleted."}
def list(self, request, params):
records = self.model.objects.all()
return self.serialize.list(records)
def nested_list(self, request, params):
getattr(self.parent_model, self.set_name)
parent = self.parent_model.objects.get(pk=params.get(self.parent_id))
records = getattr(parent, self.set_name).all()
return self.serialize.list(records)
def show(self, request, params):
record = self.model.objects.get(pk=params[self.id])
relations = self.add_relations(record)
return self.serialize.model(record, relations)
def add_relations(self, record):
relations = {}
print >>sys.stderr, "relations", self.relations
if self.relations:
for name in self.relations:
relations[name] = self.serialize.list_to_dict(getattr(record, name).all())
print >>sys.stderr, "DATA:", "add relations", getattr(record, name).all()
print >>sys.stderr, "relations", relations
return relations
def setUp(self):
self.calls = 0
self.s = Serializer()
class SimulationRunner:
def __init__(self, problem, config):
self.abs_path = os.path.dirname(os.path.abspath(__file__))
self.serializer = Serializer()
self.config_file = config
self.config = self.serializer.read_config(self.abs_path + "/" + config)
self.ik_solution_generator = IKSolutionGenerator()
self.clear_stats(problem)
self.run(problem)
def clear_stats(self, problem):
for file in glob.glob(self.abs_path + "/../problems/" + problem + "/log.log"):
os.remove(file)
for file in glob.glob(self.abs_path + "/../problems/" + problem + "/log.log"):
os.remove(file)
if os.path.isdir(self.abs_path + "/stats"):
cmd = "rm -rf " + self.abs_path + "/stats/*"
os.system(cmd)
else:
os.makedirs(self.abs_path + "/stats")
def write_rewards(self, log_file_path):
rewards = []
for line in tuple(open(log_file_path, 'r')):
if "Reward" in line:
rewards.append(float(line.split(":")[1][1:]))
n, min_max, mean_rewards, reward_variances, skew, kurt = stats.describe(np.array(rewards))
if np.isnan(np.asscalar(reward_variances)):
reward_variances = np.array(0.0)
with open(log_file_path, "a+") as f:
f.write("Reward variance: " + str(np.asscalar(reward_variances)) + " \n")
with open(log_file_path, "a+") as f:
f.write("Reward standard deviation: " + str(np.asscalar(np.sqrt(reward_variances))) + " \n")
def read_float_data(self, log_file_path, searchstring):
data = 0.0
for line in tuple(open(log_file_path, 'r')):
if searchstring in line:
data = float(line.split(":")[1].split(" ")[1].rstrip("\n"))
return data
def read_int_data(self, log_file_path, searchstring):
data = 0.0
for line in tuple(open(log_file_path, 'r')):
if searchstring in line:
data = int(float(line.split(":")[1].split(" ")[1].rstrip("\n")))
return data
def read_avg_path_length(self, log_file_path, num_runs):
l = 0
for line in tuple(open(log_file_path, 'r')):
if "S:" in line:
l += 1
return l / num_runs
def replace_option_in_config(self, option_name, value, problem, is_bool=False):
bool_str = None
if is_bool:
bool_str = "false"
if value:
bool_str = "true"
lines = list(open(self.abs_path + "/../problems/" + problem + "/default.cfg", 'r'))
idx = -1
for i in xrange(len(lines)):
if option_name in lines[i]:
idx = i
if not idx == -1:
if is_bool:
lines[idx] = option_name + " = " + str(bool_str) + "\n"
else:
lines[idx] = option_name + " = " + str(value) + "\n"
os.remove(self.abs_path + "/../problems/" + problem + "/default.cfg")
with open(self.abs_path + "/../problems/" + problem + "/default.cfg", 'a+') as f:
for line in lines:
f.write(line)
def compareEnvironmentToTmpFiles(self, problem, model_file, environment_path, environment_file):
if not os.path.exists(self.abs_path + "/tmp/" + problem):
os.makedirs(self.abs_path + "/tmp/" + problem)
return False
if not (os.path.exists(self.abs_path + '/tmp/' + problem + '/' + environment_file) and
os.path.exists(self.abs_path + '/tmp/' + problem + '/' + self.config_file) and
os.path.exists(self.abs_path + '/tmp/' + problem + '/' + model_file)):
return False
with open(self.abs_path + '/../problems/' + problem + "/environment/" + environment_file, 'r') as f1, open(self.abs_path + '/tmp/' + problem + '/' + environment_file, 'r') as f2:
missing_from_b = [
diff[2:] for diff in Differ().compare(f1.readlines(), f2.readlines())
if diff.startswith('-')
]
if len(missing_from_b) != 0:
return False
with open(self.abs_path + '/../problems/' + problem + "/model/" + model_file, 'r') as f1, open(self.abs_path + '/tmp/' + problem + '/' + model_file, 'r') as f2:
missing_from_b = [
diff[2:] for diff in Differ().compare(f1.readlines(), f2.readlines())
if diff.startswith('-')
]
if len(missing_from_b) != 0:
return False
with open(self.abs_path + "/" + self.config_file, 'r') as f1, open(self.abs_path + '/tmp/' + problem + '/' + self.config_file, 'r') as f2:
missing_from_b = [
diff[2:] for diff in Differ().compare(f1.readlines(), f2.readlines())
if diff.startswith('-')
]
for i in xrange(len(missing_from_b)):
if ("num_links" in missing_from_b[i] or
"workspace_dimensions" in missing_from_b[i] or
"goal_position" in missing_from_b[i] or
"goal_radius" in missing_from_b[i]):
return False
""" If same, use existing goalstates """
try:
shutil.copy2(self.abs_path + '/tmp/' + problem + '/goalstates.txt', self.abs_path + '/../problems/' + problem + "/goalstates.txt")
except:
return False
return True
def copyToTmp(self, problem, model_file, environment_file):
shutil.copy2(self.abs_path + '/../problems/' + problem + "/environment/" + environment_file, self.abs_path + '/tmp/' + problem + '/' + environment_file)
shutil.copy2(self.abs_path + '/../problems/' + problem + "/model/" + model_file, self.abs_path + '/tmp/' + problem + '/' + model_file)
shutil.copy2(self.abs_path + '/../problems/' + problem + "/goalstates.txt", self.abs_path + '/tmp/' + problem + '/goalstates.txt')
shutil.copy2(self.abs_path + "/" + self.config_file, self.abs_path + '/tmp/' + problem + '/' + self.config_file)
def get_average_distance_to_goal_area(self, goal_position, goal_radius, cartesian_coords):
avg_dist = 0.0
goal_pos = np.array(goal_position)
for i in xrange(len(cartesian_coords)):
cart = np.array(cartesian_coords[i])
dist = np.linalg.norm(goal_pos - cart)
if dist < goal_radius:
dist = 0.0
avg_dist += dist
if avg_dist == 0.0:
return avg_dist
return np.asscalar(avg_dist) / len(cartesian_coords)
def write_mean_num_collisions(self, log_file_path, num_runs):
num_collisions = 0.0
with open(log_file_path, 'r') as f:
for line in f:
if "Trans: Collision detected: True" in line:
num_collisions += 1
mean_num_collisions = num_collisions / num_runs
with open(log_file_path, "a+") as f:
f.write("Mean num collisions per run: " + str(mean_num_collisions) + " \n")
def run(self, problem):
alg = "alg"
if "manipulator_discrete" in problem:
alg = "abt"
elif "manipulator_continuous" in problem:
alg = "gps"
num_runs = self.config['num_simulation_runs']
init_state = self.config['init_state']
discount = self.config['discount_factor']
control_rate = self.config['control_rate']
illegal_move_penalty = self.config['illegal_move_penalty']
illegal_action_penalty = self.config['illegal_action_penalty']
step_penalty = self.config['step_penalty']
exit_reward = self.config['exit_reward']
planning_time = self.config['planning_time']
particle_count = self.config['particle_count']
num_steps = self.config['num_steps']
save_particles = self.config['save_particles']
plot_particles = self.config['plot_particles']
verbose = self.config['verbose']
verbose_rrt = self.config['verbose_rrt']
rrt_stretching_factor = self.config['rrt_stretching_factor']
histories_per_step = self.config['histories_per_step']
prune_every_step = self.config['prune_every_step']
continuous_collision = self.config['continuous_collision_check']
check_linear_path = self.config['check_linear_path']
enforce_constraints = self.config['enforce_constraints']
planner = self.config['planner']
dynamic_problem = self.config['dynamic_problem']
simulation_step_size = self.config['simulation_step_size']
planning_velocity = self.config['planning_velocity']
show_viewer = self.config['show_viewer']
max_observation_distance = self.config['max_observation_distance']
particle_replenish_timeout = self.config['particle_replenish_timeout']
state_sampling_strategy = self.config['state_sampling_strategy']
num_samples_weighted_lazy = self.config['num_samples_weighted_lazy']
gravity = self.config['gravity']
particle_plot_limit = self.config['particle_plot_limit']
num_effective_particles = self.config['num_effective_particles']
save_policy = self.config['save_policy']
load_initial_policy = self.config['load_initial_policy']
initial_planning_time = self.config['initial_planning_time']
particle_filter = self.config['particle_filter']
num_actions_per_joint = self.config['num_actions_per_joint']
environment_file = self.config['environment_file']
robot_file = self.config['robot_file']
num_generated_goal_states = self.config['num_generated_goal_states']
model_file = "test.xml"
environment_path = os.path.abspath(self.abs_path + "/../problems/" + problem + "/environment/" + environment_file)
utils = Utils()
if not os.path.exists(environment_path):
print "FILE DOES NOT EXISTS"
print "load obstacles"
obstacles = utils.loadObstaclesXML(environment_path)
print "loaded obstacles"
goal_area = v_double()
utils.loadGoalArea(environment_path, goal_area)
if len(goal_area) == 0:
print "ERROR: Your environment file doesn't define a goal area"
return False
goal_position = [goal_area[i] for i in xrange(0, 3)]
goal_radius = goal_area[3]
print goal_position
print goal_radius
logging_level = logging.WARN
if verbose:
logging_level = logging.DEBUG
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging_level)
if not os.path.exists(self.abs_path + "/../problems/" + problem + "/model/" + robot_file):
print "what"
robot = Robot(os.path.abspath(self.abs_path + "/../problems/" + problem + "/model/" + robot_file))
ja = v_double()
init_angles = [init_state[i] for i in xrange(len(init_state) / 2)]
ja[:] = init_angles
print [goal_position[i] for i in xrange(len(goal_position))]
class TestSerialize(unittest.TestCase):
@classmethod
def _clearMsgDefs(self):
# Clear all non-test messages
try:
while True:
messages.MESSAGES.pop()
except IndexError:
pass
@classmethod
def setUpClass(self):
self._clearMsgDefs()
messages.MESSAGES.append(__import__("test_complex_pb2", globals(), locals(), [], -1))
def setUp(self):
self.calls = 0
self.s = Serializer()
def test_unnamed_serialize(self):
enc = self.s.serialize("TestComplex", [("normal", ["m1", "m2"])])
def f(obj):
self.assertTrue(obj.IsInitialized())
self.assertEqual(obj.nesteds[0].normal, "normal")
self.assertEqual(obj.nesteds[0].multiple[0], "m1")
self.assertEqual(obj.nesteds[0].multiple[1], "m2")
self.calls += 1
self.s.add_handler("TestComplex", f)
self.s.unserialize(enc)
self.assertEqual(self.calls, 1)
def test_named_serialize(self):
enc = self.s.serialize("TestComplex",
nesteds = [{
"normal": "normal",
"multiple": [
"m1",
"m2"
]
}]
)
def f(obj):
self.assertTrue(obj.IsInitialized())
self.assertEqual(obj.nesteds[0].normal, "normal")
self.assertEqual(obj.nesteds[0].multiple[0], "m1")
self.assertEqual(obj.nesteds[0].multiple[1], "m2")
self.calls += 1
self.s.add_handler("TestComplex", f)
self.s.unserialize(enc)
self.assertEqual(self.calls, 1)
def test_nested_length(self):
with self.assertRaises(FieldTooLongException):
self.s.serialize("TestComplex", [("", ["longerthantwentycharacters"])])
def test_obj_onto_repeated(self):
with self.assertRaises(FieldWrongTypeException):
self.s.serialize("TestComplex", [("", ("m1",))])
def test_repeated_onto_obj(self):
with self.assertRaises(FieldWrongTypeException):
self.s.serialize("TestComplex", [["", ["m1"]]])
def test_unicode_strings(self):
sbs = '}\xc8A\xc1\x8a}D\xe8\xea\x93}'
enc = self.s.serialize("TestComplex", [(sbs, ["m1", "m2"])])
def f(obj):
self.assertTrue(obj.IsInitialized())
for i in range(len(sbs)):
self.assertEqual(ord(obj.nesteds[0].normal[i]), ord(sbs[i]))
self.assertEqual(obj.nesteds[0].multiple[0], "m1")
self.assertEqual(obj.nesteds[0].multiple[1], "m2")
self.calls += 1
self.s.add_handler("TestComplex", f)
self.s.unserialize(enc)
self.assertEqual(self.calls, 1)
def test_optional_missing(self):
enc = self.s.serialize("TestOptional")
def f(obj):
self.assertTrue(obj.IsInitialized())
self.assertTrue(not obj.field)
self.calls += 1
self.s.add_handler("TestOptional", f)
self.s.unserialize(enc)
self.assertEqual(self.calls, 1)
def test_optional_set(self):
enc = self.s.serialize("TestOptional", "test")
def f(obj):
self.assertTrue(obj.IsInitialized())
self.assertEqual(obj.field, "test")
self.calls += 1
self.s.add_handler("TestOptional", f)
self.s.unserialize(enc)
self.assertEqual(self.calls, 1)
def parse_from_string(self, serialized):
serializer = Serializer()
return serializer.deserialize(serialized, self.__class__)
class SyncObj(object):
def __init__(self, selfNodeAddr, otherNodesAddrs, conf=None):
if conf is None:
self.__conf = SyncObjConf()
else:
self.__conf = conf
self.__selfNodeAddr = selfNodeAddr
self.__otherNodesAddrs = otherNodesAddrs
self.__unknownConnections = {} # descr => _Connection
self.__raftState = _RAFT_STATE.FOLLOWER
self.__raftCurrentTerm = 0
self.__votesCount = 0
self.__raftLeader = None
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__raftLog = [] # (command, logID, term)
self.__raftLog.append((None, 1, self.__raftCurrentTerm))
self.__raftCommitIndex = 1
self.__raftLastApplied = 1
self.__raftNextIndex = {}
self.__raftMatchIndex = {}
self.__lastSerializedTime = time.time()
self.__forceLogCompaction = False
self.__socket = None
self.__resolver = DnsCachingResolver(self.__conf.dnsCacheTime, self.__conf.dnsFailCacheTime)
self.__serializer = Serializer(self.__conf.fullDumpFile, self.__conf.logCompactionBatchSize)
self.__poller = createPoller()
self.__isInitialized = False
self.__lastInitTryTime = 0
self._methodToID = {}
self._idToMethod = {}
methods = sorted([m for m in dir(self) if callable(getattr(self, m))])
for i, method in enumerate(methods):
self._methodToID[method] = i
self._idToMethod[i] = getattr(self, method)
self.__thread = None
self.__mainThread = None
self.__initialised = None
self.__commandsQueue = Queue.Queue(self.__conf.commandsQueueSize)
self.__nodes = []
self.__newAppendEntriesTime = 0
self.__commandsWaitingCommit = collections.defaultdict(list) # logID => [(termID, callback), ...]
self.__commandsLocalCounter = 0
self.__commandsWaitingReply = {} # commandLocalCounter => callback
self.__properies = set()
for key in self.__dict__:
self.__properies.add(key)
if self.__conf.autoTick:
self.__mainThread = threading.current_thread()
self.__initialised = threading.Event()
self.__thread = threading.Thread(target=SyncObj._autoTickThread, args=(weakref.proxy(self),))
self.__thread.start()
while not self.__initialised.is_set():
pass
else:
self.__initInTickThread()
def __initInTickThread(self):
try:
self.__lastInitTryTime = time.time()
self.__bind()
self.__nodes = []
for nodeAddr in self.__otherNodesAddrs:
self.__nodes.append(Node(self, nodeAddr))
self.__raftNextIndex[nodeAddr] = 0
self.__raftMatchIndex[nodeAddr] = 0
self.__needLoadDumpFile = True
self.__isInitialized = True
except:
LOG_CURRENT_EXCEPTION()
def _applyCommand(self, command, callback):
try:
self.__commandsQueue.put_nowait((command, callback))
except Queue.Full:
callback(None, FAIL_REASON.QUEUE_FULL)
def _checkCommandsToApply(self):
for i in xrange(self.__conf.maxCommandsPerTick):
if self.__raftLeader is None and self.__conf.commandsWaitLeader:
break
try:
command, callback = self.__commandsQueue.get_nowait()
except Queue.Empty:
break
requestNode, requestID = None, None
if isinstance(callback, tuple):
requestNode, requestID = callback
if self.__raftState == _RAFT_STATE.LEADER:
idx, term = self.__getCurrentLogIndex() + 1, self.__raftCurrentTerm
self.__raftLog.append((command, idx, term))
if requestNode is None:
if callback is not None:
self.__commandsWaitingCommit[idx].append((term, callback))
else:
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'log_idx': idx,
'log_term': term,
})
if not self.__conf.appendEntriesUseBatch:
self.__sendAppendEntries()
elif self.__raftLeader is not None:
if requestNode is None:
message = {
'type': 'apply_command',
'command': command,
}
if callback is not None:
self.__commandsLocalCounter += 1
self.__commandsWaitingReply[self.__commandsLocalCounter] = callback
message['request_id'] = self.__commandsLocalCounter
self.__send(self.__raftLeader, message)
else:
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': FAIL_REASON.NOT_LEADER,
})
else:
if requestNode is None:
callback(None, FAIL_REASON.MISSING_LEADER)
else:
self.__send(requestNode, {
'type': 'apply_command_response',
'request_id': requestID,
'error': FAIL_REASON.NOT_LEADER,
})
def _autoTickThread(self):
self.__initInTickThread()
self.__initialised.set()
time.sleep(0.1)
try:
while True:
if not self.__mainThread.is_alive():
break
self._onTick(self.__conf.autoTickPeriod)
except ReferenceError:
pass
def _onTick(self, timeToWait=0.0):
if not self.__isInitialized:
if time.time() >= self.__lastInitTryTime + self.__conf.bindRetryTime:
self.__initInTickThread()
if not self.__isInitialized:
time.sleep(timeToWait)
return
if self.__needLoadDumpFile:
if self.__conf.fullDumpFile is not None and os.path.isfile(self.__conf.fullDumpFile):
self.__loadDumpFile()
self.__needLoadDumpFile = False
if self.__raftState in (_RAFT_STATE.FOLLOWER, _RAFT_STATE.CANDIDATE):
if self.__raftElectionDeadline < time.time():
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__raftLeader = None
self.__raftState = _RAFT_STATE.CANDIDATE
self.__raftCurrentTerm += 1
self.__votesCount = 1
for node in self.__nodes:
node.send({
'type': 'request_vote',
'term': self.__raftCurrentTerm,
'last_log_index': self.__getCurrentLogIndex(),
'last_log_term': self.__getCurrentLogTerm(),
})
self.__onLeaderChanged()
if self.__raftState == _RAFT_STATE.LEADER:
while self.__raftCommitIndex < self.__getCurrentLogIndex():
nextCommitIndex = self.__raftCommitIndex + 1
count = 1
for node in self.__nodes:
if self.__raftMatchIndex[node.getAddress()] >= nextCommitIndex:
count += 1
if count > (len(self.__nodes) + 1) / 2:
self.__raftCommitIndex = nextCommitIndex
else:
break
if time.time() > self.__newAppendEntriesTime:
self.__sendAppendEntries()
if self.__raftCommitIndex > self.__raftLastApplied:
count = self.__raftCommitIndex - self.__raftLastApplied
entries = self.__getEntries(self.__raftLastApplied + 1, count)
for entry in entries:
currentTermID = entry[2]
subscribers = self.__commandsWaitingCommit.pop(entry[1], [])
res = self.__doApplyCommand(entry[0])
for subscribeTermID, callback in subscribers:
if subscribeTermID == currentTermID:
callback(res, FAIL_REASON.SUCCESS)
else:
callback(None, FAIL_REASON.DISCARDED)
self.__raftLastApplied += 1
self._checkCommandsToApply()
self.__tryLogCompaction()
for node in self.__nodes:
node.connectIfRequired()
self.__poller.poll(timeToWait)
def _getLastCommitIndex(self):
return self.__raftCommitIndex
def _printStatus(self):
LOG_DEBUG('self', self.__selfNodeAddr)
LOG_DEBUG('leader', self.__raftLeader)
LOG_DEBUG('partner nodes', len(self.__nodes))
for n in self.__nodes:
LOG_DEBUG(n.getAddress(), n.getStatus())
LOG_DEBUG('log len:', len(self.__raftLog))
LOG_DEBUG('log size bytes:', len(zlib.compress(cPickle.dumps(self.__raftLog, -1))))
LOG_DEBUG('last applied:', self.__raftLastApplied)
LOG_DEBUG('commit idx:', self.__raftCommitIndex)
LOG_DEBUG('next node idx:', self.__raftNextIndex)
def _forceLogCompaction(self):
self.__forceLogCompaction = True
def __doApplyCommand(self, command):
args = []
kwargs = {
'_doApply': True,
}
if not isinstance(command, tuple):
funcID = command
elif len(command) == 2:
funcID, args = command
else:
funcID, args, newKwArgs = command
kwargs.update(newKwArgs)
return self._idToMethod[funcID](*args, **kwargs)
def _onMessageReceived(self, nodeAddr, message):
if message['type'] == 'request_vote':
if message['term'] > self.__raftCurrentTerm:
self.__raftCurrentTerm = message['term']
self.__raftState = _RAFT_STATE.FOLLOWER
if self.__raftState in (_RAFT_STATE.FOLLOWER, _RAFT_STATE.CANDIDATE):
lastLogTerm = message['last_log_term']
lastLogIdx = message['last_log_index']
if message['term'] >= self.__raftCurrentTerm:
if lastLogTerm < self.__getCurrentLogTerm():
return
if lastLogTerm == self.__getCurrentLogTerm() and \
lastLogIdx < self.__getCurrentLogIndex():
return
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
self.__send(nodeAddr, {
'type': 'response_vote',
'term': message['term'],
})
if message['type'] == 'append_entries' and message['term'] >= self.__raftCurrentTerm:
self.__raftElectionDeadline = time.time() + self.__generateRaftTimeout()
if self.__raftLeader != nodeAddr:
self.__onLeaderChanged()
self.__raftLeader = nodeAddr
self.__raftCurrentTerm = message['term']
self.__raftState = _RAFT_STATE.FOLLOWER
newEntries = message.get('entries', [])
serialized = message.get('serialized', None)
leaderCommitIndex = message['commit_index']
# Regular append entries
if 'prevLogIdx' in message:
prevLogIdx = message['prevLogIdx']
prevLogTerm = message['prevLogTerm']
prevEntries = self.__getEntries(prevLogIdx)
if not prevEntries:
self.__sendNextNodeIdx(nodeAddr, reset=True)
return
if prevEntries[0][2] != prevLogTerm:
self.__deleteEntriesFrom(prevLogIdx)
self.__sendNextNodeIdx(nodeAddr, reset=True)
return
if len(prevEntries) > 1:
self.__deleteEntriesFrom(prevLogIdx + 1)
self.__raftLog += newEntries
# Install snapshot
elif serialized is not None:
if self.__serializer.setTransmissionData(serialized):
self.__loadDumpFile()
self.__sendNextNodeIdx(nodeAddr)
self.__raftCommitIndex = min(leaderCommitIndex, self.__getCurrentLogIndex())
if message['type'] == 'apply_command':
if 'request_id' in message:
self._applyCommand(message['command'], (nodeAddr, message['request_id']))
else:
self._applyCommand(message['command'], None)
if message['type'] == 'apply_command_response':
requestID = message['request_id']
error = message.get('error', None)
callback = self.__commandsWaitingReply.pop(requestID, None)
if callback is not None:
if error is not None:
callback(None, error)
else:
idx = message['log_idx']
term = message['log_term']
assert idx > self.__raftLastApplied
self.__commandsWaitingCommit[idx].append((term, callback))
if self.__raftState == _RAFT_STATE.CANDIDATE:
if message['type'] == 'response_vote' and message['term'] == self.__raftCurrentTerm:
self.__votesCount += 1
if self.__votesCount > (len(self.__nodes) + 1) / 2:
self.__onBecomeLeader()
if self.__raftState == _RAFT_STATE.LEADER:
if message['type'] == 'next_node_idx':
reset = message['reset']
nextNodeIdx = message['next_node_idx']
currentNodeIdx = nextNodeIdx - 1
if reset:
self.__raftNextIndex[nodeAddr] = nextNodeIdx
self.__raftMatchIndex[nodeAddr] = currentNodeIdx
def __sendNextNodeIdx(self, nodeAddr, reset=False):
self.__send(nodeAddr, {
'type': 'next_node_idx',
'next_node_idx': self.__getCurrentLogIndex() + 1,
'reset': reset,
})
def __generateRaftTimeout(self):
minTimeout = self.__conf.raftMinTimeout
maxTimeout = self.__conf.raftMaxTimeout
return minTimeout + (maxTimeout - minTimeout) * random.random()
def __bind(self):
self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.__socket.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.__conf.sendBufferSize)
self.__socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self.__conf.recvBufferSize)
self.__socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.__socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.__socket.setblocking(0)
host, port = self.__selfNodeAddr.split(':')
self.__socket.bind((host, int(port)))
self.__socket.listen(5)
self.__poller.subscribe(self.__socket.fileno(),
self.__onNewConnection,
POLL_EVENT_TYPE.READ | POLL_EVENT_TYPE.ERROR)
def __onNewConnection(self, localDescr, event):
if event & POLL_EVENT_TYPE.READ:
try:
sock, addr = self.__socket.accept()
sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.__conf.sendBufferSize)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self.__conf.recvBufferSize)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
sock.setblocking(0)
conn = Connection(socket=sock, timeout=self.__conf.connectionTimeout)
descr = conn.fileno()
self.__unknownConnections[descr] = conn
self.__poller.subscribe(descr,
self.__processUnknownConnections,
POLL_EVENT_TYPE.READ | POLL_EVENT_TYPE.ERROR)
except socket.error as e:
if e.errno != socket.errno.EAGAIN:
self.__isInitialized = False
LOG_WARNING('Error in main socket:' + str(e))
if event & POLL_EVENT_TYPE.ERROR:
self.__isInitialized = False
LOG_WARNING('Error in main socket')
def __getCurrentLogIndex(self):
return self.__raftLog[-1][1]
def __getCurrentLogTerm(self):
return self.__raftLog[-1][2]
def __getPrevLogIndexTerm(self, nextNodeIndex):
prevIndex = nextNodeIndex - 1
entries = self.__getEntries(prevIndex, 1)
if entries:
return prevIndex, entries[0][2]
return None, None
def __getEntries(self, fromIDx, count=None):
firstEntryIDx = self.__raftLog[0][1]
if fromIDx is None or fromIDx < firstEntryIDx:
return []
diff = fromIDx - firstEntryIDx
if count is None:
return self.__raftLog[diff:]
return self.__raftLog[diff:diff + count]
def _isLeader(self):
return self.__raftState == _RAFT_STATE.LEADER
def _getLeader(self):
return self.__raftLeader
def _getRaftLogSize(self):
return len(self.__raftLog)
def __deleteEntriesFrom(self, fromIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = fromIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog = self.__raftLog[:diff]
def __deleteEntriesTo(self, toIDx):
firstEntryIDx = self.__raftLog[0][1]
diff = toIDx - firstEntryIDx
if diff < 0:
return
self.__raftLog = self.__raftLog[diff:]
def __onBecomeLeader(self):
self.__raftLeader = self.__selfNodeAddr
self.__raftState = _RAFT_STATE.LEADER
for node in self.__nodes:
nodeAddr = node.getAddress()
self.__raftNextIndex[nodeAddr] = self.__getCurrentLogIndex() + 1
self.__raftMatchIndex[nodeAddr] = 0
self.__sendAppendEntries()
def __onLeaderChanged(self):
for id in sorted(self.__commandsWaitingReply):
self.__commandsWaitingReply[id](None, FAIL_REASON.LEADER_CHANGED)
self.__commandsWaitingReply = {}
def __sendAppendEntries(self):
self.__newAppendEntriesTime = time.time() + self.__conf.appendEntriesPeriod
startTime = time.time()
for node in self.__nodes:
nodeAddr = node.getAddress()
if not node.isConnected():
self.__serializer.cancelTransmisstion(nodeAddr)
continue
sendSingle = True
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
while nextNodeIndex <= self.__getCurrentLogIndex() or sendSingle or sendingSerialized:
if nextNodeIndex >= self.__raftLog[0][1]:
prevLogIdx, prevLogTerm = self.__getPrevLogIndexTerm(nextNodeIndex)
entries = []
if nextNodeIndex <= self.__getCurrentLogIndex():
entries = self.__getEntries(nextNodeIndex, self.__conf.appendEntriesBatchSize)
self.__raftNextIndex[nodeAddr] = entries[-1][1] + 1
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'entries': entries,
'prevLogIdx': prevLogIdx,
'prevLogTerm': prevLogTerm,
}
node.send(message)
else:
transmissionData = self.__serializer.getTransmissionData(nodeAddr)
message = {
'type': 'append_entries',
'term': self.__raftCurrentTerm,
'commit_index': self.__raftCommitIndex,
'serialized': transmissionData,
}
node.send(message)
if transmissionData is not None:
isLast = transmissionData[2]
if isLast:
self.__raftNextIndex[nodeAddr] = self.__raftLog[0][1]
sendingSerialized = False
else:
sendingSerialized = True
else:
sendingSerialized = False
nextNodeIndex = self.__raftNextIndex[nodeAddr]
sendSingle = False
delta = time.time() - startTime
if delta > self.__conf.appendEntriesPeriod:
break
def __send(self, nodeAddr, message):
for node in self.__nodes:
if node.getAddress() == nodeAddr:
node.send(message)
break
def __processUnknownConnections(self, descr, event):
conn = self.__unknownConnections[descr]
partnerNode = None
remove = False
if event & POLL_EVENT_TYPE.READ:
conn.read()
nodeAddr = conn.getMessage()
if nodeAddr is not None:
for node in self.__nodes:
if node.getAddress() == nodeAddr:
partnerNode = node
break
else:
remove = True
if event & POLL_EVENT_TYPE.ERROR:
remove = True
if remove or conn.isDisconnected():
self.__unknownConnections.pop(descr)
self.__poller.unsubscribe(descr)
conn.close()
return
if partnerNode is not None:
self.__unknownConnections.pop(descr)
assert conn.fileno() is not None
partnerNode.onPartnerConnected(conn)
def _getSelfNodeAddr(self):
return self.__selfNodeAddr
def _getConf(self):
return self.__conf
def _getResolver(self):
return self.__resolver
def _getPoller(self):
return self.__poller
def __tryLogCompaction(self):
currTime = time.time()
serializeState, serializeID = self.__serializer.checkSerializing()
if serializeState == SERIALIZER_STATE.SUCCESS:
self.__lastSerializedTime = currTime
self.__deleteEntriesTo(serializeID)
if serializeState == SERIALIZER_STATE.FAILED:
LOG_WARNING("Failed to store full dump")
if serializeState != SERIALIZER_STATE.NOT_SERIALIZING:
return
if len(self.__raftLog) <= self.__conf.logCompactionMinEntries and \
currTime - self.__lastSerializedTime <= self.__conf.logCompactionMinTime and\
not self.__forceLogCompaction:
return
self.__forceLogCompaction = False
lastAppliedEntries = self.__getEntries(self.__raftLastApplied - 1, 2)
if not lastAppliedEntries:
return
data = dict([(k, self.__dict__[k]) for k in self.__dict__.keys() if k not in self.__properies])
self.__serializer.serialize((data, lastAppliedEntries[1], lastAppliedEntries[0]), lastAppliedEntries[1][1])
def __loadDumpFile(self):
try:
data = self.__serializer.deserialize()
for k, v in data[0].iteritems():
self.__dict__[k] = v
self.__raftLog = [data[2], data[1]]
self.__raftLastApplied = data[1][1]
except:
LOG_WARNING('Failed to load full dump')
LOG_CURRENT_EXCEPTION()
def serialize_to_string(self):
serializer = Serializer()
return serializer.serialize(self)