def delete_key(self, key):
try:
result = self.hdel(settings.redis_db, key)
return result == 1
except Exception:
logger.error("[DB] Failed to delete hash key. {}".format(
traceback.format_exc(limit=5)))
def receive_token(self, request):
logger.debug(
"\n\n\n\n\n\t\t\t\t\t********************** RECEIVE_TOKEN **************************"
)
logger.debug("Received {} - {}".format(request.method, request.path))
logger.verbose("headers: {}".format(request.headers))
try:
received_hash = request.headers.get("Authorization").replace(
"Bearer ", "")
if self._validate_confirmation_hash(received_hash):
if request.is_json:
received_data = request.get_json()
logger.debug(f'Authorize response data: {received_data}')
else:
return Response(status=422)
return self.handle_receive_token(
received_data, request.headers["X-Client-Id"],
request.headers["X-Owner-Id"])
else:
logger.debug("Provided invalid confirmation hash!")
return Response(status=403)
except Exception:
logger.error("Couldn't complete processing request, {}".format(
traceback.format_exc(limit=5)))
def rename_key(self, new_key, old_key):
try:
value = self.get_key(old_key)
if not value:
logger.warning(f"[DB] Key {old_key} not found in database.")
created = False
deleted = False
if value:
created = self.set_key(new_key, value)
if not created:
logger.warning(
f"[DB] error while creating {new_key} to database.")
if created:
deleted = self.delete_key(old_key)
if not deleted:
logger.warning(
f"[DB] error while deleting {old_key} from database.")
if created and deleted:
logger.info(
f"[DB] Key {old_key} renamed to {new_key} successfully.")
return created and deleted
except Exception:
logger.error(
f"[DB] Failed to rename key {old_key} to {new_key}. {traceback.format_exc(limit=5)}"
)
async def make_requests(self, conf_data):
try:
logger.info(
f"[Polling] {threading.currentThread().getName()} starting {datetime.datetime.now()}"
)
loop = asyncio.get_event_loop()
with concurrent.futures.ThreadPoolExecutor(
max_workers=DEFAULT_THREAD_MAX_WORKERS) as executor:
futures = [
loop.run_in_executor(executor, self.send_request,
conf_data, channel_id)
for channel_id in self.db.get_channels()
]
for response in await asyncio.gather(*futures):
if response:
for resp in response:
self.implementer.polling(resp)
logger.info("[Polling] {} finishing {}".format(
threading.currentThread().getName(), datetime.datetime.now()))
except Exception:
logger.error("[Polling] Error on make_requests: {}".format(
traceback.format_exc(limit=5)))
def decode(cls, line):
"""Remove backslash escaping from line.valueDecode line, either to remove
backslash espacing, or to decode base64 encoding. The content line should
contain a ENCODING=b for base64 encoding, but Apple Addressbook seems to
export a singleton parameter of 'BASE64', which does not match the 3.0
vCard spec. If we encouter that, then we transform the parameter to
ENCODING=b"""
if line.encoded:
if 'BASE64' in line.singletonparams:
line.singletonparams.remove('BASE64')
line.encoding_param = cls.base64string
encoding = getattr(line, 'encoding_param', None)
if encoding:
if encoding == 'quoted-printable':
line.value = line.value.encode('utf-8').decode(encoding)
else:
try:
line.value = line.value.decode('base64')
except binascii.Error:
# Ignore decoding errors caused by invalid base64 values
logger.error('Failed to parse vContact value field, setting to an empty string')
line.value = ''
else:
line.value = stringToTextValues(line.value)[0]
line.encoded=False
def on_disconnect(self, client, userdata, rc):
if rc != 0:
logger.error("Mqtt - Unexpected disconnection: {}".format(
RC_LIST.get(rc)))
self.mqtt_config()
else:
logger.error("Mqtt - Expected disconnection.")
def authorize(self, request):
logger.debug(
"\n\n\n\n\n\t\t\t\t\t********************** AUTHORIZE **************************"
)
logger.debug("Received {} - {}".format(request.method, request.path))
logger.verbose("headers: {}".format(request.headers))
try:
received_hash = request.headers.get("Authorization",
"").replace("Bearer ", "")
if self._validate_confirmation_hash(received_hash):
sender = {
"channel_template_id":
request.headers["X-Channeltemplate-Id"],
"client_id": request.headers["X-Client-Id"],
"owner_id": request.headers["X-Owner-Id"]
}
data = {
"location": self.implementer.auth_requests(sender=sender)
}
return Response(response=json.dumps(data),
status=200,
mimetype="application/json")
else:
logger.debug("Provided invalid confirmation hash! {}".format(
self.confirmation_hash))
return Response(status=403)
except Exception:
logger.error("Couldn't complete processing request, {}".format(
traceback.format_exc(limit=5)))
return Response(status=403)
def on_connect(self, client, userdata, flags, rc):
try:
if rc == 0:
logger.debug("Mqtt - Connected , result code {}".format(rc))
topic = "/{api_version}/{mqtt_topic}/{client_id}/channels/#".format(
mqtt_topic=settings.mqtt_topic,
api_version=settings.api_version,
client_id=settings.client_id)
if self.subscribe:
logger.notice("Mqtt - Will subscribe to {}".format(topic))
self.mqtt_client.subscribe(topic, qos=0)
if self._on_connect_callback:
self._on_connect_callback.__call__(
**self._on_connect_callback_params)
elif 0 < rc < 6:
raise Exception(RC_LIST[rc])
except Exception:
logger.error("Mqtt Exception- {}".format(
traceback.format_exc(limit=5)))
os._exit(1)
def get_or_create_channel(self, device, channel_template, client_id):
try:
channel_id = self.db.get_channel_id(device["id"])
# Validate if still exists on Muzzley
url = "{}/channels/{}".format(settings.api_server_full, channel_id)
resp = self.session.get(url, data=None)
logger.verbose("/v3/channels/{} response code {}".format(
channel_id, resp.status_code))
if resp.status_code not in (200, 201):
channel_id = self.create_channel_id(device, channel_template,
client_id)
# Ensure persistence of manufacturer's device id (key) to channel id (field) in redis hash
self.db.set_channel_id(device["id"], channel_id, True)
logger.verbose(
"Channel added to database {}".format(channel_id))
return channel_id
except UnauthorizedException as e:
logger.error(f"{e}")
except Exception:
logger.error(
f'Error get_or_create_channel {traceback.format_exc(limit=5)}')
return None
def start(self):
"""
If polling is enabled in config file, retrieves conf for polling in implementor
"""
try:
if settings.config_polling.get('enabled') is True:
logger.info('[Polling] **** starting polling ****')
conf_data = self.implementer.get_polling_conf()
if type(conf_data) is not list:
conf_data = [conf_data]
n_processes = settings.config_polling.get(
'requests_pool', DEFAULT_THREAD_MAX_WORKERS)
self.pool_requests = ThreadPool(processes=n_processes)
self.thread = threading.Thread(target=self.worker,
args=[conf_data],
name="Polling")
self.thread.daemon = True
self.thread.start()
else:
logger.info('[Polling] **** polling is not enabled ****')
except NotImplementedError as e:
logger.error("[Polling] NotImplementedError: {}".format(e))
except Exception:
logger.alert(
f"[Polling] Unexpected exception: {traceback.format_exc(limit=5)}"
)
def callback_url_resp_check(resp, mode, callback_id, url, logger):
u"""回调URL的结果检查，根据响应确定回调是否成功.
@param resp: requests的响应,
http://docs.python-requests.org/en/master/api/#requests.Response # noqa
@param mode: 回调URL的模式，详见const.CALLBACK_URL.MODE
"""
if resp.status_code != 200:
logger.error(
"[callback request error]: "
"<callback_id>=><%s>, <url>=><%s>, <resp_code>=><%s>",
callback_id,
url,
resp.status_code,
exc_info=True)
return False
# TODO: judge according to mode
try:
json_resp = resp.json()
except ValueError:
logger.error(
"[callback request error]: "
"<callback_id>=><%s>, <url>=><%s>",
callback_id,
url,
exc_info=True)
return False
if json_resp.get('retcode') == 0:
return True
return False
def has_key(self, key):
try:
result = self.hexists(settings.redis_db, key)
return result == 1
except Exception:
logger.error("[DB] Failed to check if hash has key. {}".format(
traceback.format_exc(limit=5)))
def clear_hash(self):
try:
self.delete(settings.redis_db)
logger.notice("[DB] Redis database shutdown.")
except Exception:
logger.error("[DB] Failed to clear redis database, {}".format(
traceback.format_exc(limit=5)))
def reconfig(self):
try:
self.mqtt_client.username_pw_set(username=self.client_id,
password=self.access_token)
except Exception:
logger.error(
f"Unexpected error reconfig: {traceback.format_exc(limit=5)}")
raise
def save_n_exit(self):
""" To safely exit the opened client """
try:
self.shutdown()
logger.notice("[DB] Redis database shutdown.")
except Exception:
logger.error("[DB] Failed to shutdown redis database, {}".format(
traceback.format_exc(limit=5)))
def _read_handler(self):
"""
解析数据头
"""
if not self._stream.closed():
try:
self._stream.read_bytes(4, self._read_header_callback)
except:
logger.error("read stream error", exc_info=True)
def get_redis():
try:
return DBManager(host=settings.redis_host,
port=settings.redis_port,
decode_responses=True)
except Exception as e:
logger.error(
"[DB] Failed to connect Redis-client to Redis server, {}".format(
e))
return None
def open_session(self, app, request):
val = request.headers.get("Authorization")
if not val:
return self.session_class()
max_age = app.permanent_session_lifetime.total_seconds()
try:
data = decode_from_access_token(val)
return self.session_class(data)
except Exception, e:
logger.error(e)
return self.session_class()
def gen_access_token(user_id):
payload = {
'user_id': user_id,
'exp': datetime.datetime.utcnow() + datetime.timedelta(days=365),
'nbf': datetime.datetime.utcnow(),
}
try:
return jwt.encode(payload, config.JWT_SECRET, algorithm='HS256')
except Exception, e:
logger.error(e)
return ""
def select_device(self, request):
logger.debug(
"\n\n\n\n\n\t\t\t\t\t*******************SELECT_DEVICE****************************"
)
logger.debug("Received {} - {}".format(request.method, request.path))
logger.verbose("headers: {}".format(request.headers))
try:
received_hash = request.headers.get("Authorization",
"").replace("Bearer ", "")
if self._validate_confirmation_hash(received_hash):
if request.is_json:
payload = request.get_json()
logger.verbose(format_str(payload, is_json=True))
paired_devices = payload["channels"]
if not paired_devices:
logger.error(
"No paired devices found in request: {}".format(
payload))
else:
return Response(status=422)
owner_id = request.headers["X-Owner-Id"]
client_id = request.headers["X-Client-Id"]
channel_template = request.headers["X-Channeltemplate-Id"]
channels, credentials = self.handle_channel_requests(
client_id, owner_id, channel_template, paired_devices)
sender = {
"channel_template_id": channel_template,
"client_id": client_id,
"owner_id": owner_id
}
self.implementer.did_pair_devices(
sender=sender,
credentials=credentials,
paired_devices=paired_devices,
channels=channels)
return Response(response=json.dumps(channels),
status=200,
mimetype="application/json")
else:
logger.debug("Provided invalid confirmation hash!")
return Response(status=403)
except Exception:
logger.error("Couldn't complete processing request, {}".format(
traceback.format_exc(limit=5)))
return Response(status=403)
def gen_access_token(user_id):
payload = {
'user_id': user_id,
'exp': datetime.datetime.utcnow() + datetime.timedelta(days=365),
'nbf': datetime.datetime.utcnow(),
}
try:
return jwt.encode(payload, config.JWT_SECRET, algorithm='HS256')
except Exception, e:
logger.error(e)
return ""
def mqtt_decongif(self):
try:
self.mqtt_client.unsubscribe(
"/{api_version}/managers/{client_id}/channels/#".format(
api_version=settings.api_version,
client_id=settings.client_id))
self.mqtt_client.loop_stop()
self.mqtt_client.disconnect()
self.mqtt_client.disable_logger()
except Exception:
logger.error("Mqtt - Failed to de-configure connection {}".format(
traceback.format_exc(limit=5)))
os._exit(1)
def worker(self, conf_data):
asyncio.set_event_loop(self.loop)
loop = asyncio.get_event_loop()
while True:
logger.info('[Polling] new polling request {}'.format(
datetime.datetime.now()))
try:
loop.run_until_complete(self.make_requests(conf_data))
except Exception:
logger.error(
f'[Polling] Error on worker loop, {traceback.format_exc(limit=5)}'
)
time.sleep(self.interval)
def send_request(self, conf_data, channel_id):
try:
# validate if channel exists
credentials_list = self.db.full_query(
'credential-owners/*/channels/{}'.format(channel_id))
logger.info('[Polling] {} results found for channel_id: {}'.format(
len(credentials_list), channel_id))
for credential_dict in credentials_list: # try until we find valid credentials
cred_key = credential_dict['key']
credentials = credential_dict['value']
is_valid = self.validate_channel(cred_key)
if not is_valid:
logger.debug(
'[Polling] Invalid channel {}'.format(cred_key))
continue
# Validate if token is valid before the request
now = int(time.time())
token_expiration_date = credentials['expiration_date']
if now > token_expiration_date and not token_expiration_date == 0:
logger.debug(
"[Polling] access token expired {} - now:{}, expiration:{}"
.format(cred_key, now, token_expiration_date))
continue
resp_list = []
results = self.pool_requests.starmap(
self.get_response,
zip(conf_data, repeat(credentials), repeat(channel_id),
repeat(cred_key)))
resp_list.extend([result for result in results if result])
if resp_list:
return resp_list
except requests.exceptions.RequestException as e:
logger.error('Request Error on polling.send_request {}'.format(e))
return False
except Exception:
logger.error(
f'[Polling] Unknown error on polling.send_request {traceback.format_exc(limit=5)}'
)
logger.notice(
'[Polling] No valid credentials found for channel {}'.format(
channel_id))
return False
def quote_checkout(self, request):
logger.debug("\n\n\n\n\n\t\t\t\t\t*******************QUOTE_CHECKOUT****************************")
try:
service_id, quote_id = self._basic_quote_validation(request)
result = self.implementer.quote_checkout(service_id, quote_id)
return Response(status=200, response=json.dumps(result), mimetype="application/json")
except (ValidationException, InvalidRequestException) as e:
return Response(status=412, response=json.dumps({'text': str(e), 'code': 0}), mimetype="application/json")
except UnauthorizedException as e:
return Response(status=403, response=json.dumps({'text': str(e), 'code': 0}), mimetype="application/json")
except Exception:
logger.error("[quote_checkout] Couldn't complete processing request, {}".format(traceback.format_exc(limit=5)))
return Response(status=500, response=json.dumps({'text': "Error processing request", 'code': 0}),
mimetype="application/json")
def callback_url(callback_id,
mode,
url,
method=const.HTTP_METHOD.GET,
body=None,
logger=logger.get("cgi-log")):
u"""回调URL.
@param<callback_id>: 回调在数据库中的ID(对应的表为callback_url)
@param<mode>: 回调模式，详见const.CALLBACK_URL.MODE
@param<url>: 回调的URL
@param<method>: 回调的HTTP方法
@param<body>: 如果回调方法为POST， 则为需要POST的HTTP body
@param<logger>: 用于记日志的对象
@return: (is_sucess, # 回调是否成功
resp_code, # 回调的HTTP响应状态码
resp_body, # 回调的HTTP响应body
)
"""
http_method = requests.get
if method == const.HTTP_METHOD.POST:
http_method = requests.post
# resp_code = None
# resp_body = None
try:
if body is None:
r = http_method(url)
else:
r = http_method(url, body)
except Exception:
logger.error(
"[callback request error]: "
"<callback_id>=><%s>, <url>=><%s>",
callback_id,
url,
exc_info=True)
is_success = False
else:
# resp_code = r.status_code
# resp_body = r.content
is_success = callback_url_resp_check(r, mode, callback_id, url, logger)
return is_success
def devices_list(self, request):
logger.debug(
"\n\n\n\n\n\t\t\t\t\t********************** LIST_DEVICES **************************"
)
logger.debug("Received {} - {}".format(request.method, request.path))
logger.verbose("headers: {}".format(request.headers))
try:
received_hash = request.headers.get("Authorization",
"").replace("Bearer ", "")
if self._validate_confirmation_hash(received_hash):
credentials = self.db.get_credentials(
request.headers["X-Client-Id"],
request.headers["X-Owner-Id"])
if not credentials:
logger.error("No credentials found in database")
return Response(status=404)
sender = {
"channel_template_id":
request.headers["X-Channeltemplate-Id"],
"client_id": request.headers["X-Client-Id"],
"owner_id": request.headers["X-Owner-Id"]
}
data = self.implementer.get_devices(sender=sender,
credentials=credentials)
if not data:
logger.info("No devices found for this user")
for element in data:
if "content" not in element or ("content" in element and
not element["content"]):
element["content"] = ""
return Response(response=json.dumps(data),
status=200,
mimetype="application/json")
else:
logger.debug("Provided invalid confirmation hash!")
return Response(status=403)
except Exception:
logger.error("Couldn't complete processing request, {}".format(
traceback.format_exc(limit=5)))
return Response(status=403)
def __init__(self, queue=None, implementer=None, thread_pool=None):
super(WebhookHubDevice, self).__init__(queue, implementer, thread_pool)
try:
self.refresher = TokenRefresherManager(
implementer=self.implementer)
except Exception as e:
logger.error("Failed start TokenRefresher manager, {} {}".format(
e, traceback.format_exc(limit=5)))
self.refresher = None
try:
self.poll = PollingManager(implementer=self.implementer)
except Exception as e:
logger.error("Failed start Polling manager, {} {}".format(
e, traceback.format_exc(limit=5)))
self.poll = None
def set_props(json_obj, prop_statements: str):
for statement in prop_statements.split('\n'):
if not statement:
continue
key, json_path = parse_prop_statement(statement)
logger.info('key={},json_path={}'.format(key, json_path))
value = get_by_json_path(json_path, json_obj)
logger.info('value=' + str(value))
if isinstance(value, list):
logger.error("value只能是单个的值，不能是数组！")
return
if props.get(key) is not None:
logger.warning('props[{}]已经存在，将被覆盖。原值={}，新值={}'.format(key, props[key], str(value)))
props[key] = str(value)
if ini.debug:
with ini.props_path.open(mode='w+', encoding='utf-8') as file:
json.dump(props,file)
def check(json_obj, checkpoints: str):
"""
:return: check_result, error_msg
"""
logger.info('json_obj={}'.format(str(json_obj)))
for checkpoint in checkpoints.split('\n'):
if not checkpoint:
continue
logger.info('checkpoint={}'.format(checkpoint))
json_path, operator, expectation = parse_check_point(checkpoint)
logger.info('json_path={}, operator={}, expectation={}'.format(str(json_path), str(operator), str(expectation)))
value_by_json_path = get_by_json_path(json_path, json_obj)
logger.info('value_by_json_path={}'.format(str(value_by_json_path)))
if type(value_by_json_path) != type(eval(expectation)):
return False, '{} != {}'.format(str(type(value_by_json_path)), str(type(eval(expectation))))
if operator == '==':
if value_by_json_path != eval(expectation):
return False, '预期：{}，实际：{} != {}'.format(checkpoint,str(value_by_json_path), str(eval(expectation)))
elif operator == '!=':
if value_by_json_path == eval(expectation):
return False, '预期：{}，实际：{} == {}'.format(checkpoint,str(value_by_json_path), str(eval(expectation)))
elif operator == '<':
if value_by_json_path >= eval(expectation):
return False, '预期：{}，实际：{} >= {}'.format(checkpoint,str(value_by_json_path), str(eval(expectation)))
elif operator == '>':
if value_by_json_path <= eval(expectation):
return False, '预期：{}，实际：{} <= {}'.format(checkpoint,str(value_by_json_path), str(eval(expectation)))
elif operator == '<=':
if value_by_json_path > eval(expectation):
return False, '预期：{}，实际：{} > {}'.format(checkpoint,str(value_by_json_path), str(eval(expectation)))
elif operator == '>=':
if value_by_json_path < eval(expectation):
return False, '预期：{}，实际：{} < {}'.format(checkpoint,str(value_by_json_path), str(eval(expectation)))
elif operator == '=~':
if not isinstance(value_by_json_path, str):
return False, '{}不是字符串类型'.format(str(value_by_json_path))
if not re.search(expectation, value_by_json_path):
return False, '无法在{}中找到{}'.format(value_by_json_path, expectation)
else:
logger.error('测试用例中的check_point格式错误，operator == ' + operator)
return False, '测试用例中的check_point格式错误，operator == ' + operator
return True, None
def guess_source():
system_type = request.headers.get("X-SystemType")
app_type = request.headers.get("X-AppType")
if system_type == 'Android':
if app_type == 'user':
return const.SOURCE.ANDROID_USER
elif app_type == 'business':
return const.SOURCE.ANDROID_BUSINESS
if system_type == 'iOS':
if app_type == 'business':
return const.SOURCE.IOS_BUSINESS
if config.SOURCE == const.SOURCE.UNDEFINED:
logger.error(
"guess source failed, maybe you havn't configured it [url:%s]" %
request.url)
return config.SOURCE
def get_key(self, key):
"""To get a key"s field from hash table"""
try:
if self.hexists(settings.redis_db, key):
value = self.hget(settings.redis_db, key)
logger.debug(
"[DB] Key {} retrieved from database.".format(key))
try:
evaluated_value = ast.literal_eval(value)
except Exception:
try:
evaluated_value = json.loads(value)
except Exception:
evaluated_value = value
return evaluated_value
else:
logger.info("[DB] Key {} not found in database.".format(key))
except Exception as e:
logger.error("[DB] get_key error, {}".format(e))
def start(self):
"""
If refreshing token is enabled in config file, retrieves conf for refresh in implementor
:return:
"""
try:
if settings.config_refresh.get('enabled') is True:
logger.info('[TokenRefresher] **** starting token refresher ****')
self.thread = threading.Thread(target=self.worker,
args=[self.implementer.get_refresh_token_conf()],
name="TokenRefresh")
self.thread.daemon = True
self.thread.start()
else:
logger.info('[TokenRefresher] **** token refresher is not enabled ****')
except NotImplementedError as e:
logger.error("[TokenRefresher] NotImplementedError: {}".format(e))
except Exception:
logger.alert("[TokenRefresher] Unexpected exception: {} {}".format(traceback.format_exc(limit=5)))
def set_key(self, key, value):
"""
To set a key-field in hash table
key : key of the field
value : content of field
add_reverse : stores another value-key combination in hash to
facilitate search by value inexpensively.
"""
try:
if type(value) is dict:
value = json.dumps(value)
self.hset(settings.redis_db, key, value)
logger.debug("[DB] Key {} added/updated in database".format(key))
return True
except Exception:
logger.error("[DB] Failed to set the key at hash. {}".format(
traceback.format_exc(limit=5)))
return False
