def assert_request_equal(self, expected, real_request):
method, path = expected[:2]
if urlparse(path).scheme:
match_path = real_request["full_path"]
else:
match_path = real_request["path"]
self.assertEqual((method, path), (real_request["method"], match_path))
if len(expected) > 2:
body = expected[2]
real_request["expected"] = body
err_msg = "Body mismatch for %(method)s %(path)s, " "expected %(expected)r, and got %(body)r" % real_request
self.orig_assertEqual(body, real_request["body"], err_msg)
if len(expected) > 3:
headers = CaseInsensitiveDict(expected[3])
for key, value in headers.items():
real_request["key"] = key
real_request["expected_value"] = value
real_request["value"] = real_request["headers"].get(key)
err_msg = (
"Header mismatch on %(key)r, "
"expected %(expected_value)r and got %(value)r "
"for %(method)s %(path)s %(headers)r" % real_request
)
self.orig_assertEqual(value, real_request["value"], err_msg)
real_request["extra_headers"] = dict(
(key, value) for key, value in real_request["headers"].items() if key not in headers
)
if real_request["extra_headers"]:
self.fail(
"Received unexpected headers for %(method)s " "%(path)s, got %(extra_headers)r" % real_request
)
class Response:
"""The object that each response must be packed into before sending. Same
reason as the Request object. """
__slots__ = ['version', 'status', 'headers', 'body']
def __init__(self):
self.version = HTTP1_1
self.status = 200
self.headers = CID()
self.body = bytes()
def __repr__(self):
return '<Response %s %s>' % (self.status, self.reason)
@property
def reason(self):
"""Third argument in response status line"""
return responses[self.status]
def decode(self):
"""decode response"""
decode_msg(self)
def encode(self):
"""encode response"""
encode_msg(self)
def prepare(self) -> bytes:
"""Prepare the response for socket transmission"""
self.headers['content-length'] = str(len(self.body))
return '{version} {status} {reason}{headers}\r\n\r\n'.format(
version=self.version, status=self.status, reason=self.reason,
headers=''.join(
'\r\n%s: %s' % (k, v) for k, v in self.headers.items())
).encode() + self.body
def assert_request_equal(self, expected, real_request):
method, path = expected[:2]
if urlparse(path).scheme:
match_path = real_request['full_path']
else:
match_path = real_request['path']
self.assertEqual((method, path), (real_request['method'], match_path))
if len(expected) > 2:
body = expected[2]
real_request['expected'] = body
err_msg = 'Body mismatch for %(method)s %(path)s, ' \
'expected %(expected)r, and got %(body)r' % real_request
self.orig_assertEqual(body, real_request['body'], err_msg)
if len(expected) > 3:
headers = CaseInsensitiveDict(expected[3])
for key, value in headers.items():
real_request['key'] = key
real_request['expected_value'] = value
real_request['value'] = real_request['headers'].get(key)
err_msg = ('Header mismatch on %(key)r, '
'expected %(expected_value)r and got %(value)r '
'for %(method)s %(path)s %(headers)r' %
real_request)
self.orig_assertEqual(value, real_request['value'], err_msg)
real_request['extra_headers'] = dict(
(key, value) for key, value in real_request['headers'].items()
if key not in headers)
if real_request['extra_headers']:
self.fail('Received unexpected headers for %(method)s '
'%(path)s, got %(extra_headers)r' % real_request)
def assert_request_equal(self, expected, real_request):
method, path = expected[:2]
if urlparse(path).scheme:
match_path = real_request['full_path']
else:
match_path = real_request['path']
self.assertEqual((method, path), (real_request['method'],
match_path))
if len(expected) > 2:
body = expected[2]
real_request['expected'] = body
err_msg = 'Body mismatch for %(method)s %(path)s, ' \
'expected %(expected)r, and got %(body)r' % real_request
self.orig_assertEqual(body, real_request['body'], err_msg)
if len(expected) > 3:
headers = CaseInsensitiveDict(expected[3])
for key, value in headers.items():
real_request['key'] = key
real_request['expected_value'] = value
real_request['value'] = real_request['headers'].get(key)
err_msg = (
'Header mismatch on %(key)r, '
'expected %(expected_value)r and got %(value)r '
'for %(method)s %(path)s %(headers)r' % real_request)
self.orig_assertEqual(value, real_request['value'],
err_msg)
real_request['extra_headers'] = dict(
(key, value) for key, value in real_request['headers'].items()
if key not in headers)
if real_request['extra_headers']:
self.fail('Received unexpected headers for %(method)s '
'%(path)s, got %(extra_headers)r' % real_request)
class Request:
"""The object that the server must pack all requests into. This is
necessary to support multiple search apis."""
__slots__ = ['method', 'url', 'version', 'headers', 'body']
def __init__(self):
self.version = HTTP1_1
self.headers = CID()
self.url = None # type: URL
self.body = bytes()
self.method = str()
def __repr__(self):
return '<Request %s %s>' % (self.url, self.method)
def decode(self):
"""decode request"""
decode_msg(self)
def encode(self):
"""encode request"""
encode_msg(self)
def prepare(self) -> bytes:
"""Prepare the request for socket transmission"""
self.headers['content-length'] = str(len(self.body))
headers = ''.join('\r\n%s: %s' % (k, v)
for k, v in self.headers.items())
return '{method} {url} {version}{headers}\r\n\r\n'.format(
method=self.method,
url=str(self.url),
version=self.version,
headers=headers).encode() + self.body
def test_preserve_last_key_case(self):
cid = CaseInsensitiveDict({"Accept": "application/json", "user-Agent": "requests"})
cid.update({"ACCEPT": "application/json"})
cid["USER-AGENT"] = "requests"
keyset = frozenset(["ACCEPT", "USER-AGENT"])
assert frozenset(i[0] for i in cid.items()) == keyset
assert frozenset(cid.keys()) == keyset
assert frozenset(cid) == keyset
def set_extra_headers(self, headers):
header_dict = CaseInsensitiveDict(headers)
if 'Reply-To' in header_dict:
self.data["ReplyTo"] = header_dict.pop('Reply-To')
self.data["Headers"] = [
{"Name": key, "Value": value}
for key, value in header_dict.items()
]
def set_extra_headers(self, headers):
header_dict = CaseInsensitiveDict(headers)
if 'Reply-To' in header_dict:
self.data["ReplyTo"] = header_dict.pop('Reply-To')
self.data["Headers"] = [
{"Name": key, "Value": value}
for key, value in header_dict.items()
]
def test_preserve_key_case(self):
cid = CaseInsensitiveDict({
'Accept': 'application/json',
'user-Agent': 'requests',
})
keyset = frozenset(['Accept', 'user-Agent'])
assert frozenset(i[0] for i in cid.items()) == keyset
assert frozenset(cid.keys()) == keyset
assert frozenset(cid) == keyset
def test_preserve_key_case(self):
cid = CaseInsensitiveDict({
'Accept': 'application/json',
'user-Agent': 'requests',
})
keyset = frozenset(['Accept', 'user-Agent'])
assert frozenset(i[0] for i in cid.items()) == keyset
assert frozenset(cid.keys()) == keyset
assert frozenset(cid) == keyset
def to_list(self,
json_string: str,
filters: CaseInsensitiveDict = None) -> list:
"""TODO"""
ret_list = []
for value in json.loads(json_string).values():
if not filters or all(k in value and value[k] == v
for k, v in filters.items()):
ret_list.append(self.row_to_entity(value))
return ret_list
class SSDPResponse(object):
"""A container class for received SSDP responses."""
def __init__(self, response):
"""Initializes the SSDPResponse instance based on a response string"""
super(BasicSSDPServiceDiscoverer.SSDPResponse, self).__init__()
# Initialize fields
self.Location = None
self.USN = None
self.ST = None
self.Headers = CaseInsensitiveDict()
# Parse response
self._fromString(response)
def _fromString(self, str):
"""Parses a response string and assigns values to the SSDPResponse object.
:param str str: The string to parse."""
# Lazy method to parse all http-headers
h = CaseInsensitiveDict({k.lower(): v for k, v in dict(re.findall(r'(?P<name>.*?): (?P<value>.*?)\r\n', str)).items()})
self.Headers = h
# Set major fields
if 'location' in h:
self.Location = h['location']
if 'USN' in h:
self.USN = h['USN']
if 'ST' in h:
self.ST = h['ST']
def __repr__(self):
return '<SSDPResponse from %s at %s; Headers: %s>' % (self.USN, self.Location, self.Headers.__repr__())
def __hash__(self):
if self.USN is not None:
return hash(self.USN)
return hash(tuple(self.Headers.items()))
def __eq__(self, other):
if self is not None and other is None:
return False
if not isinstance(other, self.__class__):
return False
return hash(self) == hash(other)
def __ne__(self, other):
return not self.__eq__(other)
def clean_response_headers(self, headers: structures.CaseInsensitiveDict) -> dict:
new_headers = dict(headers)
ignored_headers = self.ignored_response_headers
for key, value in headers.items():
if key.lower() in ignored_headers:
continue
new_headers[key] = value
return new_headers
def test_preserve_last_key_case(self):
cid = CaseInsensitiveDict({
'Accept': 'application/json',
'user-Agent': 'requests',
})
cid.update({'ACCEPT': 'application/json'})
cid['USER-AGENT'] = 'requests'
keyset = frozenset(['ACCEPT', 'USER-AGENT'])
assert frozenset(i[0] for i in cid.items()) == keyset
assert frozenset(cid.keys()) == keyset
assert frozenset(cid) == keyset
def test_preserve_last_key_case(self):
cid = CaseInsensitiveDict({
'Accept': 'application/json',
'user-Agent': 'requests',
})
cid.update({'ACCEPT': 'application/json'})
cid['USER-AGENT'] = 'requests'
keyset = frozenset(['ACCEPT', 'USER-AGENT'])
assert frozenset(i[0] for i in cid.items()) == keyset
assert frozenset(cid.keys()) == keyset
assert frozenset(cid) == keyset
def verify_signature(self, query_parameters):
"""Verify the signature provided with the query parameters.
http://docs.shopify.com/api/tutorials/oauth
example usage::
from shopify_trois import Credentials
from shopify_trois.engines import Json as Shopify
from urllib.parse import parse_qsl
credentials = Credentials(
api_key='your_api_key',
scope=['read_orders'],
secret='your_app_secret'
)
shopify = Shopify(shop_name="your_store_name", credentials=\
credentials)
query_parameters = parse_qsl("code=238420989938cb70a609f6ece2e2586\
b&shop=yourstore.myshopify.com×tamp=1373382939&\
signature=6fb122e33c21851c465345b8cb97245e")
if not shopify.verify_signature(query_parameters):
raise Exception("invalid signature")
credentials.code = dict(query_parameters).get('code')
shopify.setup_access_token()
:returns: Returns True if the signature is valid.
"""
params = CaseInsensitiveDict(query_parameters)
signature = params.pop("signature", None)
calculated = ["%s=%s" % (k, v) for k, v in params.items()]
calculated.sort()
calculated = "".join(calculated)
calculated = "{secret}{calculated}".format(
secret=self.credentials.secret,
calculated=calculated
)
md5 = hashlib.md5()
md5.update(calculated.encode('utf-8'))
produced = md5.hexdigest()
return produced == signature
def verify_signature(self, query_parameters):
"""Verify the signature provided with the query parameters.
http://docs.shopify.com/api/tutorials/oauth
example usage::
from shopify_trois import Credentials
from shopify_trois.engines import Json as Shopify
from urllib.parse import parse_qsl
credentials = Credentials(
api_key='your_api_key',
scope=['read_orders'],
secret='your_app_secret'
)
shopify = Shopify(shop_name="your_store_name", credentials=\
credentials)
query_parameters = parse_qsl("code=238420989938cb70a609f6ece2e2586\
b&shop=yourstore.myshopify.com×tamp=1373382939&\
signature=6fb122e33c21851c465345b8cb97245e")
if not shopify.verify_signature(query_parameters):
raise Exception("invalid signature")
credentials.code = dict(query_parameters).get('code')
shopify.setup_access_token()
:returns: Returns True if the signature is valid.
"""
params = CaseInsensitiveDict(query_parameters)
signature = params.pop("signature", None)
calculated = ["%s=%s" % (k, v) for k, v in params.items()]
calculated.sort()
calculated = "".join(calculated)
calculated = "{secret}{calculated}".format(
secret=self.credentials.secret, calculated=calculated)
md5 = hashlib.md5()
md5.update(calculated.encode('utf-8'))
produced = md5.hexdigest()
return produced == signature
def _prepare_request(self, command, json, opcode_name, fetch_list,
**kwargs):
params = CaseInsensitiveDict(**kwargs)
params.update({
'apiKey': self.key,
opcode_name: command,
})
if json:
params['response'] = 'json'
if 'page' in kwargs or fetch_list:
params.setdefault('pagesize', PAGE_SIZE)
kind = 'params' if self.method == 'get' else 'data'
return kind, {k: v for k, v in params.items()}
def dict_subset(input_list,
dic_data: CaseInsensitiveDict) -> CaseInsensitiveDict:
for name in input_list:
if name not in [key.strip().lower() for key in dic_data.keys()]:
logging.warning(
f"provided entry with the name - {name} is not "
f"valid The program will disregard this input")
input_list.remove(name)
sub_data = {
key: value
for key, value in dic_data.items() if key.strip().lower() in
[item.strip().lower() for item in input_list]
}
return CaseInsensitiveDict(data=sub_data)
def test_inject_into_non_sampled_context(self):
carrier = CaseInsensitiveDict()
AwsXRayPropagatorTest.XRAY_PROPAGATOR.inject(
carrier,
build_test_current_context(),
)
injected_items = set(carrier.items())
expected_items = set(
CaseInsensitiveDict({
TRACE_HEADER_KEY:
"Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0"
}).items())
self.assertEqual(injected_items, expected_items)
def process_headers(
self,
headers: CaseInsensitiveDict = None,
content_type: ContentType = ContentType.APPLICATION_JSON,
content_length: int = 0) -> None:
"""TODO"""
headers = MockServerHandler.remove_reserved_headers(headers)
if headers and len(headers) > 0:
for key, value in headers.items():
self.send_header(key, value)
self.send_header("Server",
'MockServer v{}'.format(self.parent.version))
self.send_header("Date", self.date_time_string())
self.send_header("Content-Type", str(content_type))
self.send_header("Content-Length", str(content_length))
self.end_headers()
def test_inject_into_context_with_non_default_state(self):
carrier = CaseInsensitiveDict()
AwsXRayPropagatorTest.XRAY_PROPAGATOR.inject(
carrier,
build_test_current_context(trace_state=TraceState([("foo",
"bar")])),
)
# TODO: (NathanielRN) Assert trace state when the propagator supports it
injected_items = set(carrier.items())
expected_items = set(
CaseInsensitiveDict({
TRACE_HEADER_KEY:
"Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0"
}).items())
self.assertEqual(injected_items, expected_items)
def _user_preferences(self, _users_data: CaseInsensitiveDict) -> Dict:
food_user_mappings = list()
user_drinks_mappings = dict()
for _user, pref in _users_data.items():
for item, val in pref.items():
if item == 'wont_eat':
food_user_mappings.append(
{key.strip().lower(): _user.strip()
for key in val})
elif item == 'drinks':
user_drinks_mappings[_user.strip()] = set(
v.strip().lower() for v in val)
return {
'bad_food_user_mappings': self._merge_dicts(food_user_mappings),
'user_drinks_mappings': user_drinks_mappings
}
def _prepare_request(self, command, json=True, opcode_name='command',
fetch_list=False, **kwargs):
params = CaseInsensitiveDict(**kwargs)
params.update({
'apiKey': self.key,
opcode_name: command,
})
if json:
params['response'] = 'json'
if 'page' in kwargs or fetch_list:
params.setdefault('pagesize', PAGE_SIZE)
if 'expires' not in params and self.expiration.total_seconds() >= 0:
params['signatureVersion'] = '3'
tz = pytz.utc
expires = tz.localize(datetime.utcnow() + self.expiration)
params['expires'] = expires.astimezone(tz).strftime(EXPIRES_FORMAT)
kind = 'params' if self.method == 'get' else 'data'
return kind, dict(params.items())
class UserStore:
"""
Simple in-memory store for user info. Attribute names are case-insensitive.
Users can be retrieved by id or username
"""
def __init__(self):
self.users = CaseInsensitiveDict() # key = id
self.names = CaseInsensitiveDict() # key = username
def add(self, user_info):
self._add_default_attributes(user_info)
self.users[user_info.get('id')] = dict(user_info)
self.names[user_info.get('username')] = dict(user_info)
return id
def get_by_id(self, id):
return self.users.get(id, None)
def get_by_name(self, username):
return self.names.get(username, None)
def update_scopes(self, username, scopes):
self.names[username]['consented_scopes'] += ' ' + scopes
def list(self):
"""
Returns a list of dictionaries representing users.
password and consented_scopes attributes are not returned
"""
return [self._copy_user(u[1]) for u in self.users.items()]
def _copy_user(self, user):
d = copy.deepcopy(dict(user))
self._del_default_attributes(d)
return d
def _add_default_attributes(self, user_info):
if 'consented_scopes' not in user_info:
user_info['consented_scopes'] = ''
def _del_default_attributes(self, dictionary):
del dictionary['consented_scopes']
del dictionary['password']
def _get_email_headers_from_part(self, part):
email_headers = list(part.items())
if not email_headers:
return {}
# Convert the header tuple into a dictionary
headers = CaseInsensitiveDict()
# assume duplicate header names with unique values. ex: received
for x in email_headers:
try:
headers.setdefault(x[0].lower().replace('-', '_').replace(' ', '_'), []).append(x[1])
except Exception as e:
error_msg = self._get_error_message_from_exception(e)
err = "Error occurred while converting the header tuple into a dictionary"
self.debug_print("{}. {}".format(err, error_msg))
headers = {k.lower(): '\n'.join(v) for k, v in headers.items()}
return dict(headers)
class Response:
"""The object that each response must be packed into before sending. Same
reason as the Request object. """
__slots__ = ['version', 'status', 'headers', 'body']
def __init__(self):
self.version = HTTP1_1
self.status = 200
self.headers = CID()
self.body = bytes()
def __repr__(self):
return '<Response %s %s>' % (self.status, self.reason)
@property
def reason(self):
"""Third argument in response status line"""
return responses[self.status]
def unpack(self):
"""Preprocess http message (e.g decode)"""
if self.headers.get('content-encoding', '') == 'gzip':
self.body = gzip.decompress(self.body)
def prepare(self, request: Request) -> bytes:
"""Prepare the response for socket transmission"""
if 'gzip' in request.headers.get('accept-encoding', ''):
self.headers['content-encoding'] = 'gzip'
self.body = gzip.compress(self.body)
self.headers['content-length'] = str(len(self.body))
return '{version} {status} {reason}{headers}\r\n\r\n'.format(
version=self.version,
status=self.status,
reason=self.reason,
headers=''.join(
'\r\n%s: %s' % (k, v)
for k, v in self.headers.items())).encode() + self.body
def _prepare_request(self,
command,
json=True,
opcode_name='command',
fetch_list=False,
**kwargs):
params = CaseInsensitiveDict(**kwargs)
params.update({
'apiKey': self.key,
opcode_name: command,
})
if json:
params['response'] = 'json'
if 'page' in kwargs or fetch_list:
params.setdefault('pagesize', PAGE_SIZE)
if 'expires' not in params and self.expiration.total_seconds() >= 0:
params['signatureVersion'] = '3'
tz = pytz.utc
expires = tz.localize(datetime.utcnow() + self.expiration)
params['expires'] = expires.astimezone(tz).strftime(EXPIRES_FORMAT)
kind = 'params' if self.method == 'get' else 'data'
return kind, dict(params.items())
def sniffit():
"""
Perform an HTTP/HTTPS request to the address that the user specifid
:return:
TODO Make the Google Verification a separate module with annotion
"""
parsed_url = urlparse(request.json["url"])
app.logger.info(request.remote_addr + " " + parsed_url.netloc)
# Processing the headers to be sent to the URL that the user defined in the interface.
# What we are doing here is making sure the the user can't override some headers that we want to force such as
# X-Forwarded-For.
request_headers = CaseInsensitiveDict({header["key"]: header["value"] for header in request.json["headers"]})
request_headers["X-Forwarded-For"] = request.remote_addr
request_headers["X-Anti-Abuse"] = app.config.get("ABUSE_CONTACT")
request_headers = {string.capwords(k, "-"): v for (k, v) in request_headers.items()}
# Request Parameters
if type(request.json["parameters"]) is list:
request_parameters = "&".join([cgi.escape(header["key"])+"="+cgi.escape(header["value"]) for header in request.json["parameters"]])
else:
request_parameters = request.json["parameters"]
# Base Response JSON
response_json = {'success': False, 'sniffed': None, 'messages': []}
try:
if string.lower(request.json["method"]) in ["get", "head", "options"]:
response = requests.request(request.json["method"], request.json["url"], verify=False,
params=request_parameters, headers=request_headers)
else:
response = requests.request(request.json["method"], request.json["url"],
verify=False, data=request_parameters, headers=request_headers)
# I prefer to have the capitalized headers in the frontend
# This will convert the headers from 'content-type' to 'Content-Type'
response_headers = {string.capwords(k, "-"): v for (k, v) in response.headers.items()}
# This is for the adrministrators only so there is no need for the end-user to see this
request_headers.pop("X-Anti-Abuse")
request_headers.pop("X-Forwarded-For")
# Create a history of redirects to inform the user
redirections = [{"url": redirect.headers["location"]} for redirect in response.history]
# Geo Location
ipaddress = socket.gethostbyname(parsed_url.netloc)
geolocation_response = requests.get("http://ip-api.com/json/" + ipaddress);
response_json["success"] = True
response_json["showRecaptcha"] = recaptcha_handler.is_token_invalid()
response_json["sniffed"] = {
'headers': {
'response': response_headers,
'request': request_headers
},
'ipaddress': ipaddress,
'geolocation': geolocation_response.json(),
'ssl': None,
'redirect': redirections,
'body': base64.b64encode(cgi.escape(response.text.encode("UTF-8"))),
'size': response.headers.get("content-length", False),
'ssize': len(response.text.encode("UTF-8")),
'elapsed': response.elapsed.total_seconds(),
'status': {
"reason": response.reason,
"code": str(response.status_code)
}
}
except Exception as e:
raise RequestFailedException(repr(e))
return jsonify(response_json)
class Blink:
"""Class to initialize communication."""
def __init__(
self,
username=None,
password=None,
cred_file=None,
refresh_rate=DEFAULT_REFRESH,
motion_interval=DEFAULT_MOTION_INTERVAL,
legacy_subdomain=False,
no_prompt=False,
persist_key=None,
device_id="Blinkpy",
):
"""
Initialize Blink system.
:param username: Blink username (usually email address)
:param password: Blink password
:param cred_file: JSON formatted file to store credentials.
If username and password are given, file
is ignored. Otherwise, username and password
are loaded from file.
:param refresh_rate: Refresh rate of blink information.
Defaults to 15 (seconds)
:param motion_interval: How far back to register motion in minutes.
Defaults to last refresh time.
Useful for preventing motion_detected property
from de-asserting too quickly.
:param legacy_subdomain: Set to TRUE to use old 'rest.region'
endpoints (only use if you are having
api issues).
:param no_prompt: Set to TRUE if using an implementation that needs to
suppress command-line output.
:param persist_key: Location of persistant identifier.
:param device_id: Identifier for the application. Default is 'Blinkpy'.
This is used when logging in and should be changed to
fit the implementation (ie. "Home Assistant" in a
Home Assistant integration).
"""
self.login_handler = LoginHandler(
username=username,
password=password,
cred_file=cred_file,
persist_key=persist_key,
device_id=device_id,
)
self._token = None
self._auth_header = None
self._host = None
self.account_id = None
self.client_id = None
self.network_ids = []
self.urls = None
self.sync = CaseInsensitiveDict({})
self.region = None
self.region_id = None
self.last_refresh = None
self.refresh_rate = refresh_rate
self.session = create_session()
self.networks = []
self.cameras = CaseInsensitiveDict({})
self.video_list = CaseInsensitiveDict({})
self.login_url = LOGIN_URLS[0]
self.login_urls = []
self.motion_interval = motion_interval
self.version = __version__
self.legacy = legacy_subdomain
self.no_prompt = no_prompt
self.available = False
self.key_required = False
self.login_response = {}
@property
def auth_header(self):
"""Return the authentication header."""
return self._auth_header
def start(self):
"""
Perform full system setup.
Method logs in and sets auth token, urls, and ids for future requests.
Essentially this is just a wrapper function for ease of use.
"""
if not self.available:
self.get_auth_token()
if self.key_required and not self.no_prompt:
email = self.login_handler.data["username"]
key = input("Enter code sent to {}: ".format(email))
result = self.login_handler.send_auth_key(self, key)
self.key_required = not result
self.setup_post_verify()
elif not self.key_required:
self.setup_post_verify()
def setup_post_verify(self):
"""Initialize blink system after verification."""
camera_list = self.get_cameras()
networks = self.get_ids()
for network_name, network_id in networks.items():
if network_id not in camera_list.keys():
camera_list[network_id] = {}
_LOGGER.warning("No cameras found for %s", network_name)
sync_module = BlinkSyncModule(
self, network_name, network_id, camera_list[network_id]
)
sync_module.start()
self.sync[network_name] = sync_module
self.cameras = self.merge_cameras()
self.available = self.refresh()
self.key_required = False
def login(self):
"""Perform server login. DEPRECATED."""
_LOGGER.warning(
"Method is deprecated and will be removed in a future version. Please use the LoginHandler.login() method instead."
)
return self.login_handler.login(self)
def get_auth_token(self, is_retry=False):
"""Retrieve the authentication token from Blink."""
self.login_response = self.login_handler.login(self)
if not self.login_response:
self.available = False
return False
self.setup_params(self.login_response)
if self.login_handler.check_key_required(self):
self.key_required = True
return self._auth_header
def setup_params(self, response):
"""Retrieve blink parameters from login response."""
self.login_url = self.login_handler.login_url
((self.region_id, self.region),) = response["region"].items()
self._host = "{}.{}".format(self.region_id, BLINK_URL)
self._token = response["authtoken"]["authtoken"]
self._auth_header = {"Host": self._host, "TOKEN_AUTH": self._token}
self.urls = BlinkURLHandler(self.region_id, legacy=self.legacy)
self.networks = self.get_networks()
self.client_id = response["client"]["id"]
self.account_id = response["account"]["id"]
def get_networks(self):
"""Get network information."""
response = api.request_networks(self)
try:
return response["summary"]
except KeyError:
return None
def get_ids(self):
"""Set the network ID and Account ID."""
all_networks = []
network_dict = {}
for network, status in self.networks.items():
if status["onboarded"]:
all_networks.append("{}".format(network))
network_dict[status["name"]] = network
self.network_ids = all_networks
return network_dict
def get_cameras(self):
"""Retrieve a camera list for each onboarded network."""
response = api.request_homescreen(self)
try:
all_cameras = {}
for camera in response["cameras"]:
camera_network = str(camera["network_id"])
camera_name = camera["name"]
camera_id = camera["id"]
camera_info = {"name": camera_name, "id": camera_id}
if camera_network not in all_cameras:
all_cameras[camera_network] = []
all_cameras[camera_network].append(camera_info)
return all_cameras
except KeyError:
_LOGGER.error("Initialization failue. Could not retrieve cameras.")
return {}
@Throttle(seconds=MIN_THROTTLE_TIME)
def refresh(self, force_cache=False):
"""
Perform a system refresh.
:param force_cache: Force an update of the camera cache
"""
if self.check_if_ok_to_update() or force_cache:
for sync_name, sync_module in self.sync.items():
_LOGGER.debug("Attempting refresh of sync %s", sync_name)
sync_module.refresh(force_cache=force_cache)
if not force_cache:
# Prevents rapid clearing of motion detect property
self.last_refresh = int(time.time())
return True
return False
def check_if_ok_to_update(self):
"""Check if it is ok to perform an http request."""
current_time = int(time.time())
last_refresh = self.last_refresh
if last_refresh is None:
last_refresh = 0
if current_time >= (last_refresh + self.refresh_rate):
return True
return False
def merge_cameras(self):
"""Merge all sync camera dicts into one."""
combined = CaseInsensitiveDict({})
for sync in self.sync:
combined = merge_dicts(combined, self.sync[sync].cameras)
return combined
def download_videos(self, path, since=None, camera="all", stop=10, debug=False):
"""
Download all videos from server since specified time.
:param path: Path to write files. /path/<cameraname>_<recorddate>.mp4
:param since: Date and time to get videos from.
Ex: "2018/07/28 12:33:00" to retrieve videos since
July 28th 2018 at 12:33:00
:param camera: Camera name to retrieve. Defaults to "all".
Use a list for multiple cameras.
:param stop: Page to stop on (~25 items per page. Default page 10).
:param debug: Set to TRUE to prevent downloading of items.
Instead of downloading, entries will be printed to log.
"""
if since is None:
since_epochs = self.last_refresh
else:
parsed_datetime = parse(since, fuzzy=True)
since_epochs = parsed_datetime.timestamp()
formatted_date = get_time(time_to_convert=since_epochs)
_LOGGER.info("Retrieving videos since %s", formatted_date)
if not isinstance(camera, list):
camera = [camera]
for page in range(1, stop):
response = api.request_videos(self, time=since_epochs, page=page)
_LOGGER.debug("Processing page %s", page)
try:
result = response["media"]
if not result:
raise IndexError
except (KeyError, IndexError):
_LOGGER.info("No videos found on page %s. Exiting.", page)
break
self._parse_downloaded_items(result, camera, path, debug)
def _parse_downloaded_items(self, result, camera, path, debug):
"""Parse downloaded videos."""
for item in result:
try:
created_at = item["created_at"]
camera_name = item["device_name"]
is_deleted = item["deleted"]
address = item["media"]
except KeyError:
_LOGGER.info("Missing clip information, skipping...")
continue
if camera_name not in camera and "all" not in camera:
_LOGGER.debug("Skipping videos for %s.", camera_name)
continue
if is_deleted:
_LOGGER.debug("%s: %s is marked as deleted.", camera_name, address)
continue
clip_address = "{}{}".format(self.urls.base_url, address)
filename = "{}-{}".format(camera_name, created_at)
filename = "{}.mp4".format(slugify(filename))
filename = os.path.join(path, filename)
if not debug:
if os.path.isfile(filename):
_LOGGER.info("%s already exists, skipping...", filename)
continue
response = api.http_get(self, url=clip_address, stream=True, json=False)
with open(filename, "wb") as vidfile:
copyfileobj(response.raw, vidfile)
_LOGGER.info("Downloaded video to %s", filename)
else:
print(
("Camera: {}, Timestamp: {}, " "Address: {}, Filename: {}").format(
camera_name, created_at, address, filename
)
)
def test_preserve_key_case(self):
cid = CaseInsensitiveDict({"Accept": "application/json", "user-Agent": "requests"})
keyset = frozenset(["Accept", "user-Agent"])
assert frozenset(i[0] for i in cid.items()) == keyset
assert frozenset(cid.keys()) == keyset
assert frozenset(cid) == keyset
class ScrapeConfig:
PUBLIC_DATACENTER_POOL = 'public_datacenter_pool'
PUBLIC_RESIDENTIAL_POOL = 'public_residential_pool'
url: str
retry: bool = True
method: str = 'GET'
country: Optional[str] = None
render_js: bool = False
cache: bool = False
cache_clear: bool = False
ssl: bool = False
dns: bool = False
asp: bool = False
debug: bool = False
raise_on_upstream_error: bool = True
cache_ttl: Optional[int] = None
proxy_pool: Optional[str] = None
session: Optional[str] = None
tags: Optional[List[str]] = None
correlation_id: Optional[str] = None
cookies: Optional[CaseInsensitiveDict] = None
body: Optional[str] = None
data: Optional[Dict] = None
headers: Optional[CaseInsensitiveDict] = None
graphql: Optional[str] = None
js: str = None
rendering_wait: int = None
wait_for_selector: Optional[str] = None
session_sticky_proxy: bool = True
screenshots: Optional[Dict] = None
webhook: Optional[str] = None
def __init__(self,
url: str,
retry: bool = True,
method: str = 'GET',
country: Optional[str] = None,
render_js: bool = False,
cache: bool = False,
cache_clear: bool = False,
ssl: bool = False,
dns: bool = False,
asp: bool = False,
debug: bool = False,
raise_on_upstream_error: bool = True,
cache_ttl: Optional[int] = None,
proxy_pool: Optional[str] = None,
session: Optional[str] = None,
tags: Optional[Set[str]] = None,
correlation_id: Optional[str] = None,
cookies: Optional[CaseInsensitiveDict] = None,
body: Optional[str] = None,
data: Optional[Dict] = None,
headers: Optional[Union[CaseInsensitiveDict,
Dict[str, str]]] = None,
graphql: Optional[str] = None,
js: str = None,
rendering_wait: int = None,
wait_for_selector: Optional[str] = None,
screenshots: Optional[Dict] = None,
session_sticky_proxy: Optional[bool] = None,
webhook: Optional[str] = None):
assert (type(url) is str)
if isinstance(tags, List):
tags = set(tags)
cookies = cookies or {}
headers = headers or {}
self.cookies = CaseInsensitiveDict(cookies)
self.headers = CaseInsensitiveDict(headers)
self.url = url
self.retry = retry
self.method = method
self.country = country
self.session_sticky_proxy = session_sticky_proxy
self.render_js = render_js
self.cache = cache
self.cache_clear = cache_clear
self.asp = asp
self.webhook = webhook
self.session = session
self.debug = debug
self.cache_ttl = cache_ttl
self.proxy_pool = proxy_pool
self.tags = tags or set()
self.correlation_id = correlation_id
self.wait_for_selector = wait_for_selector
self.body = body
self.data = data
self.graphql = graphql
self.js = js
self.rendering_wait = rendering_wait
self.raise_on_upstream_error = raise_on_upstream_error
self.screenshots = screenshots
self.key = None
self.dns = dns
self.ssl = ssl
if cookies:
_cookies = []
for name, value in cookies.items():
_cookies.append(name + '=' + value)
if 'cookie' in self.headers:
if self.headers['cookie'][-1] != ';':
self.headers['cookie'] += ';'
else:
self.headers['cookie'] = ''
self.headers['cookie'] += '; '.join(_cookies)
if self.body and self.data:
raise ScrapeConfigError(
'You cannot pass both parameters body and data. You must choose'
)
if method in ['POST', 'PUT', 'PATCH']:
if self.body is None and self.data is not None:
if 'content-type' not in self.headers:
self.headers[
'content-type'] = 'application/x-www-form-urlencoded'
self.body = urlencode(data)
else:
if self.headers['content-type'].find(
'application/json') != -1:
self.body = json.dumps(data)
elif self.headers['content-type'].find(
'application/x-www-form-urlencoded') != -1:
self.body = urlencode(data)
else:
raise ScrapeConfigError(
'Content-Type "%s" not supported, use body parameter to pass pre encoded body according to your content type'
% self.headers['content-type'])
elif self.body is None and self.data is None:
self.headers['content-type'] = 'text/plain'
def _bool_to_http(self, _bool: bool) -> str:
return 'true' if _bool is True else 'false'
def generate_distributed_correlation_id(self):
self.correlation_id = abs(
hash('-'.join(
[gethostname(),
str(getpid()),
str(currentThread().ident)])))
def to_api_params(self, key: str) -> Dict:
params = {
'key': self.key if self.key is not None else key,
'url': quote(self.url)
}
if self.country is not None:
params['country'] = self.country
for name, value in self.headers.items():
params['headers[%s]' % name] = value
if self.webhook is not None:
params['webhook_name'] = self.webhook
if self.render_js is True:
params['render_js'] = self._bool_to_http(self.render_js)
if self.wait_for_selector is not None:
params['wait_for_selector'] = self.wait_for_selector
if self.js:
params['js'] = b64encode(
self.js.encode('utf-8')).decode('utf-8')
if self.rendering_wait:
params['rendering_wait'] = self.rendering_wait
if self.screenshots is not None:
for name, element in self.screenshots.items():
params['screenshots[%s]' % name] = element
else:
if self.wait_for_selector is not None:
logging.warning(
'Params "wait_for_selector" is ignored. Works only if render_js is enabled'
)
if self.screenshots:
logging.warning(
'Params "screenshots" is ignored. Works only if render_js is enabled'
)
if self.js:
logging.warning(
'Params "js" is ignored. Works only if render_js is enabled'
)
if self.rendering_wait:
logging.warning(
'Params "rendering_wait" is ignored. Works only if render_js is enabled'
)
if self.asp is True:
params['asp'] = self._bool_to_http(self.asp)
if self.retry is False:
params['retry'] = self._bool_to_http(self.retry)
if self.cache is True:
params['cache'] = self._bool_to_http(self.cache)
if self.cache_clear is True:
params['cache_clear'] = self._bool_to_http(self.cache_clear)
if self.cache_ttl is not None:
params['cache_ttl'] = self.cache_ttl
else:
if self.cache_clear is True:
logging.warning(
'Params "cache_clear" is ignored. Works only if cache is enabled'
)
if self.cache_ttl is not None:
logging.warning(
'Params "cache_ttl" is ignored. Works only if cache is enabled'
)
if self.dns is True:
params['dns'] = self._bool_to_http(self.dns)
if self.ssl is True:
params['ssl'] = self._bool_to_http(self.ssl)
if self.tags:
params['tags'] = ','.join(self.tags)
if self.correlation_id:
params['correlation_id'] = self.correlation_id
if self.session:
params['session'] = self.session
if self.session_sticky_proxy is True: # false by default
params['session_sticky_proxy'] = self._bool_to_http(
self.session_sticky_proxy)
else:
if self.session_sticky_proxy:
logging.warning(
'Params "session_sticky_proxy" is ignored. Works only if session is enabled'
)
if self.debug is True:
params['debug'] = self._bool_to_http(self.debug)
if self.graphql:
params['graphql_query'] = quote(self.graphql)
if self.proxy_pool is not None:
params['proxy_pool'] = self.proxy_pool
return params
@staticmethod
def from_exported_config(config: str) -> 'ScrapeConfig':
try:
from msgpack import loads as msgpack_loads
except ImportError as e:
print(
'You must install msgpack package - run: pip install "scrapfly-sdk[seepdup] or pip install msgpack'
)
raise
data = msgpack_loads(base64.b64decode(config))
headers = {}
for name, value in data['headers'].items():
if isinstance(value, Iterable):
headers[name] = '; '.join(value)
else:
headers[name] = value
return ScrapeConfig(url=data['url'],
retry=data['retry'],
headers=headers,
session=data['session'],
session_sticky_proxy=data['session_sticky_proxy'],
cache=data['cache'],
cache_ttl=data['cache_ttl'],
cache_clear=data['cache_clear'],
render_js=data['render_js'],
method=data['method'],
asp=data['asp'],
body=data['body'],
ssl=data['ssl'],
dns=data['dns'],
country=data['country'],
debug=data['debug'],
correlation_id=data['correlation_id'],
tags=data['tags'],
graphql=data['graphql_query'],
js=data['js'],
rendering_wait=data['rendering_wait'],
screenshots=data['screenshots'] or {},
proxy_pool=data['proxy_pool'])
class Blink():
"""Class to initialize communication."""
def __init__(self, username=None, password=None,
refresh_rate=REFRESH_RATE):
"""
Initialize Blink system.
:param username: Blink username (usually email address)
:param password: Blink password
:param refresh_rate: Refresh rate of blink information.
Defaults to 15 (seconds)
"""
self._username = username
self._password = password
self._token = None
self._auth_header = None
self._host = None
self.account_id = None
self.network_ids = []
self.urls = None
self.sync = CaseInsensitiveDict({})
self.region = None
self.region_id = None
self.last_refresh = None
self.refresh_rate = refresh_rate
self.session = None
self.networks = []
self.cameras = CaseInsensitiveDict({})
self._login_url = LOGIN_URL
@property
def auth_header(self):
"""Return the authentication header."""
return self._auth_header
def start(self):
"""
Perform full system setup.
Method logs in and sets auth token, urls, and ids for future requests.
Essentially this is just a wrapper function for ease of use.
"""
if self._username is None or self._password is None:
self.login()
else:
self.get_auth_token()
networks = self.get_ids()
for network_name, network_id in networks.items():
sync_module = BlinkSyncModule(self, network_name, network_id)
sync_module.start()
self.sync[network_name] = sync_module
self.cameras = self.merge_cameras()
def login(self):
"""Prompt user for username and password."""
self._username = input("Username:"******"Password:"******"Login successful!")
return True
_LOGGER.warning("Unable to login with %s.", self._username)
return False
def get_auth_token(self):
"""Retrieve the authentication token from Blink."""
if not isinstance(self._username, str):
raise BlinkAuthenticationException(ERROR.USERNAME)
if not isinstance(self._password, str):
raise BlinkAuthenticationException(ERROR.PASSWORD)
login_url = LOGIN_URL
self.session = create_session()
response = api.request_login(self,
login_url,
self._username,
self._password)
if response.status_code == 200:
response = response.json()
(self.region_id, self.region), = response['region'].items()
else:
_LOGGER.debug(
("Received response code %s "
"when authenticating, "
"trying new url"), response.status_code
)
login_url = LOGIN_BACKUP_URL
response = api.request_login(self,
login_url,
self._username,
self._password)
self.region_id = 'piri'
self.region = "UNKNOWN"
self._host = "{}.{}".format(self.region_id, BLINK_URL)
self._token = response['authtoken']['authtoken']
self._auth_header = {'Host': self._host,
'TOKEN_AUTH': self._token}
self.networks = response['networks']
self.urls = BlinkURLHandler(self.region_id)
self._login_url = login_url
return self._auth_header
def get_ids(self):
"""Set the network ID and Account ID."""
response = api.request_networks(self)
# Look for only onboarded network, flag warning if multiple
# since it's unexpected
all_networks = []
network_dict = {}
for network, status in self.networks.items():
if status['onboarded']:
all_networks.append('{}'.format(network))
network_dict[status['name']] = network
# For the first onboarded network we find, grab the account id
for resp in response['networks']:
if str(resp['id']) in all_networks:
self.account_id = resp['account_id']
break
self.network_ids = all_networks
return network_dict
def refresh(self, force_cache=False):
"""
Perform a system refresh.
:param force_cache: Force an update of the camera cache
"""
if self.check_if_ok_to_update() or force_cache:
for sync_name, sync_module in self.sync.items():
_LOGGER.debug("Attempting refresh of sync %s", sync_name)
sync_module.refresh(force_cache=force_cache)
def check_if_ok_to_update(self):
"""Check if it is ok to perform an http request."""
current_time = int(time.time())
last_refresh = self.last_refresh
if last_refresh is None:
last_refresh = 0
if current_time >= (last_refresh + self.refresh_rate):
self.last_refresh = current_time
return True
return False
def merge_cameras(self):
"""Merge all sync camera dicts into one."""
combined = CaseInsensitiveDict({})
for sync in self.sync:
combined = merge_dicts(combined, self.sync[sync].cameras)
return combined
class tizgmusicproxy(object):
"""A class for logging into a Google Play Music account and retrieving song
URLs.
"""
all_songs_album_title = "All Songs"
thumbs_up_playlist_name = "Thumbs Up"
def __init__(self, email, password, device_id):
self.__gmusic = Mobileclient()
self.__email = email
self.__device_id = device_id
self.logged_in = False
self.queue = list()
self.queue_index = -1
self.play_queue_order = list()
self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"])
self.current_play_mode = self.play_modes.NORMAL
self.now_playing_song = None
userdir = os.path.expanduser('~')
tizconfig = os.path.join(userdir, ".config/tizonia/." + email + ".auth_token")
auth_token = ""
if os.path.isfile(tizconfig):
with open(tizconfig, "r") as f:
auth_token = pickle.load(f)
if auth_token:
# 'Keep track of the auth token' workaround. See:
# https://github.com/diraimondo/gmusicproxy/issues/34#issuecomment-147359198
print_msg("[Google Play Music] [Authenticating] : " \
"'with cached auth token'")
self.__gmusic.android_id = device_id
self.__gmusic.session._authtoken = auth_token
self.__gmusic.session.is_authenticated = True
try:
self.__gmusic.get_registered_devices()
except CallFailure:
# The token has expired. Reset the client object
print_wrn("[Google Play Music] [Authenticating] : " \
"'auth token expired'")
self.__gmusic = Mobileclient()
auth_token = ""
if not auth_token:
attempts = 0
print_nfo("[Google Play Music] [Authenticating] : " \
"'with user credentials'")
while not self.logged_in and attempts < 3:
self.logged_in = self.__gmusic.login(email, password, device_id)
attempts += 1
with open(tizconfig, "a+") as f:
f.truncate()
pickle.dump(self.__gmusic.session._authtoken, f)
self.library = CaseInsensitiveDict()
self.song_map = CaseInsensitiveDict()
self.playlists = CaseInsensitiveDict()
self.stations = CaseInsensitiveDict()
def logout(self):
""" Reset the session to an unauthenticated, default state.
"""
self.__gmusic.logout()
def set_play_mode(self, mode):
""" Set the playback mode.
:param mode: curren tvalid values are "NORMAL" and "SHUFFLE"
"""
self.current_play_mode = getattr(self.play_modes, mode)
self.__update_play_queue_order()
def current_song_title_and_artist(self):
""" Retrieve the current track's title and artist name.
"""
logging.info("current_song_title_and_artist")
song = self.now_playing_song
if song:
title = to_ascii(self.now_playing_song.get('title'))
artist = to_ascii(self.now_playing_song.get('artist'))
logging.info("Now playing %s by %s", title, artist)
return artist, title
else:
return '', ''
def current_song_album_and_duration(self):
""" Retrieve the current track's album and duration.
"""
logging.info("current_song_album_and_duration")
song = self.now_playing_song
if song:
album = to_ascii(self.now_playing_song.get('album'))
duration = to_ascii \
(self.now_playing_song.get('durationMillis'))
logging.info("album %s duration %s", album, duration)
return album, int(duration)
else:
return '', 0
def current_track_and_album_total(self):
"""Return the current track number and the total number of tracks in the
album, if known.
"""
logging.info("current_track_and_album_total")
song = self.now_playing_song
track = 0
total = 0
if song:
try:
track = self.now_playing_song['trackNumber']
total = self.now_playing_song['totalTrackCount']
logging.info("track number %s total tracks %s", track, total)
except KeyError:
logging.info("trackNumber or totalTrackCount : not found")
else:
logging.info("current_song_track_number_"
"and_total_tracks : not found")
return track, total
def current_song_year(self):
""" Return the current track's year of publication.
"""
logging.info("current_song_year")
song = self.now_playing_song
year = 0
if song:
try:
year = song['year']
logging.info("track year %s", year)
except KeyError:
logging.info("year : not found")
else:
logging.info("current_song_year : not found")
return year
def clear_queue(self):
""" Clears the playback queue.
"""
self.queue = list()
self.queue_index = -1
def enqueue_artist(self, arg):
""" Search the user's library for tracks from the given artist and adds
them to the playback queue.
:param arg: an artist
"""
try:
self.__update_local_library()
artist = None
if arg not in self.library.keys():
for name, art in self.library.iteritems():
if arg.lower() in name.lower():
artist = art
print_wrn("[Google Play Music] '{0}' not found. " \
"Playing '{1}' instead." \
.format(arg.encode('utf-8'), \
name.encode('utf-8')))
break
if not artist:
# Play some random artist from the library
random.seed()
artist = random.choice(self.library.keys())
artist = self.library[artist]
print_wrn("[Google Play Music] '{0}' not found. "\
"Feeling lucky?." \
.format(arg.encode('utf-8')))
else:
artist = self.library[arg]
tracks_added = 0
for album in artist:
tracks_added += self.__enqueue_tracks(artist[album])
print_wrn("[Google Play Music] Playing '{0}'." \
.format(to_ascii(artist)))
self.__update_play_queue_order()
except KeyError:
raise KeyError("Artist not found : {0}".format(arg))
def enqueue_album(self, arg):
""" Search the user's library for albums with a given name and adds
them to the playback queue.
"""
try:
self.__update_local_library()
album = None
artist = None
tentative_album = None
tentative_artist = None
for library_artist in self.library:
for artist_album in self.library[library_artist]:
print_nfo("[Google Play Music] [Album] '{0}'." \
.format(to_ascii(artist_album)))
if not album:
if arg.lower() == artist_album.lower():
album = artist_album
artist = library_artist
break
if not tentative_album:
if arg.lower() in artist_album.lower():
tentative_album = artist_album
tentative_artist = library_artist
if album:
break
if not album and tentative_album:
album = tentative_album
artist = tentative_artist
print_wrn("[Google Play Music] '{0}' not found. " \
"Playing '{1}' instead." \
.format(arg.encode('utf-8'), \
album.encode('utf-8')))
if not album:
# Play some random album from the library
random.seed()
artist = random.choice(self.library.keys())
album = random.choice(self.library[artist].keys())
print_wrn("[Google Play Music] '{0}' not found. "\
"Feeling lucky?." \
.format(arg.encode('utf-8')))
if not album:
raise KeyError("Album not found : {0}".format(arg))
self.__enqueue_tracks(self.library[artist][album])
print_wrn("[Google Play Music] Playing '{0}'." \
.format(to_ascii(album)))
self.__update_play_queue_order()
except KeyError:
raise KeyError("Album not found : {0}".format(arg))
def enqueue_playlist(self, arg):
"""Search the user's library for playlists with a given name
and adds the tracks of the first match to the playback queue.
Requires Unlimited subscription.
"""
try:
self.__update_local_library()
self.__update_playlists()
self.__update_playlists_unlimited()
playlist = None
playlist_name = None
for name, plist in self.playlists.items():
print_nfo("[Google Play Music] [Playlist] '{0}'." \
.format(to_ascii(name)))
if arg not in self.playlists.keys():
for name, plist in self.playlists.iteritems():
if arg.lower() in name.lower():
playlist = plist
playlist_name = name
print_wrn("[Google Play Music] '{0}' not found. " \
"Playing '{1}' instead." \
.format(arg.encode('utf-8'), \
to_ascii(name)))
break
if not playlist:
# Play some random playlist from the library
random.seed()
playlist_name = random.choice(self.playlists.keys())
playlist = self.playlists[playlist_name]
print_wrn("[Google Play Music] '{0}' not found. "\
"Feeling lucky?." \
.format(arg.encode('utf-8')))
else:
playlist_name = arg
playlist = self.playlists[arg]
self.__enqueue_tracks(playlist)
print_wrn("[Google Play Music] Playing '{0}'." \
.format(to_ascii(playlist_name)))
self.__update_play_queue_order()
except KeyError:
raise KeyError("Playlist not found : {0}".format(arg))
def enqueue_station_unlimited(self, arg):
"""Search the user's library for a station with a given name
and add its tracks to the playback queue.
Requires Unlimited subscription.
"""
try:
# First try to find a suitable station in the user's library
self.__enqueue_user_station_unlimited(arg)
if not len(self.queue):
# If no suitable station is found in the user's library, then
# search google play unlimited for a potential match.
self.__enqueue_station_unlimited(arg)
if not len(self.queue):
raise KeyError
except KeyError:
raise KeyError("Station not found : {0}".format(arg))
def enqueue_genre_unlimited(self, arg):
"""Search Unlimited for a genre with a given name and add its
tracks to the playback queue.
Requires Unlimited subscription.
"""
print_msg("[Google Play Music] [Retrieving genres] : '{0}'. " \
.format(self.__email))
try:
all_genres = list()
root_genres = self.__gmusic.get_genres()
second_tier_genres = list()
for root_genre in root_genres:
second_tier_genres += self.__gmusic.get_genres(root_genre['id'])
all_genres += root_genres
all_genres += second_tier_genres
for genre in all_genres:
print_nfo("[Google Play Music] [Genre] '{0}'." \
.format(to_ascii(genre['name'])))
genre = dict()
if arg not in all_genres:
genre = next((g for g in all_genres \
if arg.lower() in to_ascii(g['name']).lower()), \
None)
tracks_added = 0
while not tracks_added:
if not genre and len(all_genres):
# Play some random genre from the search results
random.seed()
genre = random.choice(all_genres)
print_wrn("[Google Play Music] '{0}' not found. "\
"Feeling lucky?." \
.format(arg.encode('utf-8')))
genre_name = genre['name']
genre_id = genre['id']
station_id = self.__gmusic.create_station(genre_name, \
None, None, None, genre_id)
num_tracks = 200
tracks = self.__gmusic.get_station_tracks(station_id, num_tracks)
tracks_added = self.__enqueue_tracks(tracks)
logging.info("Added %d tracks from %s to queue", tracks_added, genre_name)
if not tracks_added:
# This will produce another iteration in the loop
genre = None
print_wrn("[Google Play Music] Playing '{0}'." \
.format(to_ascii(genre['name'])))
self.__update_play_queue_order()
except KeyError:
raise KeyError("Genre not found : {0}".format(arg))
except CallFailure:
raise RuntimeError("Operation requires an Unlimited subscription.")
def enqueue_situation_unlimited(self, arg):
"""Search Unlimited for a situation with a given name and add its
tracks to the playback queue.
Requires Unlimited subscription.
"""
print_msg("[Google Play Music] [Retrieving situations] : '{0}'. " \
.format(self.__email))
try:
self.__enqueue_situation_unlimited(arg)
if not len(self.queue):
raise KeyError
logging.info("Added %d tracks from %s to queue", \
len(self.queue), arg)
except KeyError:
raise KeyError("Situation not found : {0}".format(arg))
except CallFailure:
raise RuntimeError("Operation requires an Unlimited subscription.")
def enqueue_artist_unlimited(self, arg):
"""Search Unlimited for an artist and adds the artist's 200 top tracks to the
playback queue.
Requires Unlimited subscription.
"""
try:
artist = self.__gmusic_search(arg, 'artist')
include_albums = False
max_top_tracks = 200
max_rel_artist = 0
artist_tracks = dict()
if artist:
artist_tracks = self.__gmusic.get_artist_info \
(artist['artist']['artistId'],
include_albums, max_top_tracks,
max_rel_artist)['topTracks']
if not artist_tracks:
raise KeyError
tracks_added = self.__enqueue_tracks(artist_tracks)
logging.info("Added %d tracks from %s to queue", \
tracks_added, arg)
self.__update_play_queue_order()
except KeyError:
raise KeyError("Artist not found : {0}".format(arg))
except CallFailure:
raise RuntimeError("Operation requires an Unlimited subscription.")
def enqueue_album_unlimited(self, arg):
"""Search Unlimited for an album and add its tracks to the
playback queue.
Requires Unlimited subscription.
"""
try:
album = self.__gmusic_search(arg, 'album')
album_tracks = dict()
if album:
album_tracks = self.__gmusic.get_album_info \
(album['album']['albumId'])['tracks']
if not album_tracks:
raise KeyError
print_wrn("[Google Play Music] Playing '{0}'." \
.format((album['album']['name']).encode('utf-8')))
tracks_added = self.__enqueue_tracks(album_tracks)
logging.info("Added %d tracks from %s to queue", \
tracks_added, arg)
self.__update_play_queue_order()
except KeyError:
raise KeyError("Album not found : {0}".format(arg))
except CallFailure:
raise RuntimeError("Operation requires an Unlimited subscription.")
def enqueue_tracks_unlimited(self, arg):
""" Search Unlimited for a track name and adds all the matching tracks
to the playback queue.
Requires Unlimited subscription.
"""
print_msg("[Google Play Music] [Retrieving library] : '{0}'. " \
.format(self.__email))
try:
max_results = 200
track_hits = self.__gmusic.search(arg, max_results)['song_hits']
if not len(track_hits):
# Do another search with an empty string
track_hits = self.__gmusic.search("", max_results)['song_hits']
print_wrn("[Google Play Music] '{0}' not found. "\
"Feeling lucky?." \
.format(arg.encode('utf-8')))
tracks = list()
for hit in track_hits:
tracks.append(hit['track'])
tracks_added = self.__enqueue_tracks(tracks)
logging.info("Added %d tracks from %s to queue", \
tracks_added, arg)
self.__update_play_queue_order()
except KeyError:
raise KeyError("Playlist not found : {0}".format(arg))
except CallFailure:
raise RuntimeError("Operation requires an Unlimited subscription.")
def enqueue_promoted_tracks_unlimited(self):
""" Retrieve the url of the next track in the playback queue.
"""
try:
tracks = self.__gmusic.get_promoted_songs()
count = 0
for track in tracks:
store_track = self.__gmusic.get_track_info(track['storeId'])
if u'id' not in store_track.keys():
store_track[u'id'] = store_track['nid']
self.queue.append(store_track)
count += 1
if count == 0:
print_wrn("[Google Play Music] Operation requires " \
"an Unlimited subscription.")
logging.info("Added %d Unlimited promoted tracks to queue", \
count)
self.__update_play_queue_order()
except CallFailure:
raise RuntimeError("Operation requires an Unlimited subscription.")
def next_url(self):
""" Retrieve the url of the next track in the playback queue.
"""
if len(self.queue):
self.queue_index += 1
if (self.queue_index < len(self.queue)) \
and (self.queue_index >= 0):
next_song = self.queue[self.play_queue_order[self.queue_index]]
return self.__retrieve_track_url(next_song)
else:
self.queue_index = -1
return self.next_url()
else:
return ''
def prev_url(self):
""" Retrieve the url of the previous track in the playback queue.
"""
if len(self.queue):
self.queue_index -= 1
if (self.queue_index < len(self.queue)) \
and (self.queue_index >= 0):
prev_song = self.queue[self.play_queue_order[self.queue_index]]
return self.__retrieve_track_url(prev_song)
else:
self.queue_index = len(self.queue)
return self.prev_url()
else:
return ''
def __update_play_queue_order(self):
""" Update the queue playback order.
A sequential order is applied if the current play mode is "NORMAL" or a
random order if current play mode is "SHUFFLE"
"""
total_tracks = len(self.queue)
if total_tracks:
if not len(self.play_queue_order):
# Create a sequential play order, if empty
self.play_queue_order = range(total_tracks)
if self.current_play_mode == self.play_modes.SHUFFLE:
random.shuffle(self.play_queue_order)
print_nfo("[Google Play Music] [Tracks in queue] '{0}'." \
.format(total_tracks))
def __retrieve_track_url(self, song):
""" Retrieve a song url
"""
song_url = self.__gmusic.get_stream_url(song['id'], self.__device_id)
try:
self.now_playing_song = song
return song_url
except AttributeError:
logging.info("Could not retrieve the song url!")
raise
def __update_local_library(self):
""" Retrieve the songs and albums from the user's library
"""
print_msg("[Google Play Music] [Retrieving library] : '{0}'. " \
.format(self.__email))
songs = self.__gmusic.get_all_songs()
self.playlists[self.thumbs_up_playlist_name] = list()
# Retrieve the user's song library
for song in songs:
if "rating" in song and song['rating'] == "5":
self.playlists[self.thumbs_up_playlist_name].append(song)
song_id = song['id']
song_artist = song['artist']
song_album = song['album']
self.song_map[song_id] = song
if song_artist == "":
song_artist = "Unknown Artist"
if song_album == "":
song_album = "Unknown Album"
if song_artist not in self.library:
self.library[song_artist] = CaseInsensitiveDict()
self.library[song_artist][self.all_songs_album_title] = list()
if song_album not in self.library[song_artist]:
self.library[song_artist][song_album] = list()
self.library[song_artist][song_album].append(song)
self.library[song_artist][self.all_songs_album_title].append(song)
# Sort albums by track number
for artist in self.library.keys():
logging.info("Artist : %s", to_ascii(artist))
for album in self.library[artist].keys():
logging.info(" Album : %s", to_ascii(album))
if album == self.all_songs_album_title:
sorted_album = sorted(self.library[artist][album],
key=lambda k: k['title'])
else:
sorted_album = sorted(self.library[artist][album],
key=lambda k: k.get('trackNumber',
0))
self.library[artist][album] = sorted_album
def __update_stations_unlimited(self):
""" Retrieve stations (Unlimited)
"""
self.stations.clear()
stations = self.__gmusic.get_all_stations()
self.stations[u"I'm Feeling Lucky"] = 'IFL'
for station in stations:
station_name = station['name']
logging.info("station name : %s", to_ascii(station_name))
self.stations[station_name] = station['id']
def __enqueue_user_station_unlimited(self, arg):
""" Enqueue a user station (Unlimited)
"""
print_msg("[Google Play Music] [Station search "\
"in user's library] : '{0}'. " \
.format(self.__email))
self.__update_stations_unlimited()
station_name = arg
station_id = None
for name, st_id in self.stations.iteritems():
print_nfo("[Google Play Music] [Station] '{0}'." \
.format(to_ascii(name)))
if arg not in self.stations.keys():
for name, st_id in self.stations.iteritems():
if arg.lower() in name.lower():
station_id = st_id
station_name = name
break
else:
station_id = self.stations[arg]
num_tracks = 200
tracks = list()
if station_id:
try:
tracks = self.__gmusic.get_station_tracks(station_id, \
num_tracks)
except KeyError:
raise RuntimeError("Operation requires an "
"Unlimited subscription.")
tracks_added = self.__enqueue_tracks(tracks)
if tracks_added:
if arg != station_name:
print_wrn("[Google Play Music] '{0}' not found. " \
"Playing '{1}' instead." \
.format(arg.encode('utf-8'), name.encode('utf-8')))
logging.info("Added %d tracks from %s to queue", tracks_added, arg)
self.__update_play_queue_order()
else:
print_wrn("[Google Play Music] '{0}' has no tracks. " \
.format(station_name))
if not len(self.queue):
print_wrn("[Google Play Music] '{0}' " \
"not found in the user's library. " \
.format(arg.encode('utf-8')))
def __enqueue_station_unlimited(self, arg, max_results=200, quiet=False):
"""Search for a station and enqueue all of its tracks (Unlimited)
"""
if not quiet:
print_msg("[Google Play Music] [Station search in "\
"Google Play Music] : '{0}'. " \
.format(arg.encode('utf-8')))
try:
station_name = arg
station_id = None
station = self.__gmusic_search(arg, 'station', max_results, quiet)
if station:
station = station['station']
station_name = station['name']
seed = station['seed']
seed_type = seed['seedType']
track_id = seed['trackId'] if seed_type == u'2' else None
artist_id = seed['artistId'] if seed_type == u'3' else None
album_id = seed['albumId'] if seed_type == u'4' else None
genre_id = seed['genreId'] if seed_type == u'5' else None
playlist_token = seed['playlistShareToken'] if seed_type == u'8' else None
curated_station_id = seed['curatedStationId'] if seed_type == u'9' else None
num_tracks = max_results
tracks = list()
try:
station_id \
= self.__gmusic.create_station(station_name, \
track_id, \
artist_id, \
album_id, \
genre_id, \
playlist_token, \
curated_station_id)
tracks \
= self.__gmusic.get_station_tracks(station_id, \
num_tracks)
except KeyError:
raise RuntimeError("Operation requires an "
"Unlimited subscription.")
tracks_added = self.__enqueue_tracks(tracks)
if tracks_added:
if not quiet:
print_wrn("[Google Play Music] [Station] : '{0}'." \
.format(station_name.encode('utf-8')))
logging.info("Added %d tracks from %s to queue", \
tracks_added, arg.encode('utf-8'))
self.__update_play_queue_order()
except KeyError:
raise KeyError("Station not found : {0}".format(arg))
def __enqueue_situation_unlimited(self, arg):
"""Search for a situation and enqueue all of its tracks (Unlimited)
"""
print_msg("[Google Play Music] [Situation search in "\
"Google Play Music] : '{0}'. " \
.format(arg.encode('utf-8')))
try:
situation_hits = self.__gmusic.search(arg)['situation_hits']
if not len(situation_hits):
# Do another search with an empty string
situation_hits = self.__gmusic.search("")['situation_hits']
print_wrn("[Google Play Music] '{0}' not found. "\
"Feeling lucky?." \
.format(arg.encode('utf-8')))
situation = next((hit for hit in situation_hits \
if 'best_result' in hit.keys()), None)
num_tracks = 200
if not situation and len(situation_hits):
max_results = num_tracks / len(situation_hits)
for hit in situation_hits:
situation = hit['situation']
print_nfo("[Google Play Music] [Situation] '{0} : {1}'." \
.format((hit['situation']['title']).encode('utf-8'),
(hit['situation']['description']).encode('utf-8')))
self.__enqueue_station_unlimited(situation['title'], max_results, True)
if not situation:
raise KeyError
except KeyError:
raise KeyError("Situation not found : {0}".format(arg))
def __enqueue_tracks(self, tracks):
""" Add tracks to the playback queue
"""
count = 0
for track in tracks:
if u'id' not in track.keys():
track[u'id'] = track['nid']
self.queue.append(track)
count += 1
return count
def __update_playlists(self):
""" Retrieve the user's playlists
"""
plists = self.__gmusic.get_all_user_playlist_contents()
for plist in plists:
plist_name = plist['name']
logging.info("playlist name : %s", to_ascii(plist_name))
tracks = plist['tracks']
tracks.sort(key=itemgetter('creationTimestamp'))
self.playlists[plist_name] = list()
for track in tracks:
try:
song = self.song_map[track['trackId']]
self.playlists[plist_name].append(song)
except IndexError:
pass
def __update_playlists_unlimited(self):
""" Retrieve shared playlists (Unlimited)
"""
plists_subscribed_to = [p for p in self.__gmusic.get_all_playlists() \
if p.get('type') == 'SHARED']
for plist in plists_subscribed_to:
share_tok = plist['shareToken']
playlist_items \
= self.__gmusic.get_shared_playlist_contents(share_tok)
plist_name = plist['name']
logging.info("shared playlist name : %s", to_ascii(plist_name))
self.playlists[plist_name] = list()
for item in playlist_items:
try:
song = item['track']
song['id'] = item['trackId']
self.playlists[plist_name].append(song)
except IndexError:
pass
def __gmusic_search(self, query, query_type, max_results=200, quiet=False):
""" Search Google Play (Unlimited)
"""
search_results = self.__gmusic.search(query, max_results)[query_type + '_hits']
result = next((hit for hit in search_results \
if 'best_result' in hit.keys()), None)
if not result and len(search_results):
secondary_hit = None
for hit in search_results:
if not quiet:
print_nfo("[Google Play Music] [{0}] '{1}'." \
.format(query_type.capitalize(),
(hit[query_type]['name']).encode('utf-8')))
if query.lower() == \
to_ascii(hit[query_type]['name']).lower():
result = hit
break
if query.lower() in \
to_ascii(hit[query_type]['name']).lower():
secondary_hit = hit
if not result and secondary_hit:
result = secondary_hit
if not result and not len(search_results):
# Do another search with an empty string
search_results = self.__gmusic.search("")[query_type + '_hits']
if not result and len(search_results):
# Play some random result from the search results
random.seed()
result = random.choice(search_results)
if not quiet:
print_wrn("[Google Play Music] '{0}' not found. "\
"Feeling lucky?." \
.format(query.encode('utf-8')))
return result
def complete(current, previous, step, previous_step_metas, table_concordance=None, anteprevious=None, debug=False):
current = copy.deepcopy(current)
previous = copy.deepcopy(previous)
table_concordance = CaseInsensitiveDict(table_concordance or dict())
DEBUG = debug or '--debug' in sys.argv
def log(text):
if DEBUG:
print(text, file=sys.stderr)
def exit():
raise Exception('[complete_articles] Fatal error')
find_num = re.compile(r'-?[a-z]*(\d+)(-[ta][\dIV]+|\D)?$')
oldnum = 0
oldstep = {}
oldjson = []
oldstatus = {}
oldartids = []
oldarts = []
oldsects = []
olddepot = None
try:
for line in previous:
if line["type"] != "texte":
oldjson.append(line)
else:
oldnum = int(find_num.search(line['id']).group(1))
olddepot = line['depot']
if line["type"] == "article":
keys = list(line['alineas'].keys())
keys.sort()
oldstep[line["titre"]] = [line['alineas'][k] for k in keys]
oldstatus[line["titre"]] = line['statut']
oldartids.append(line["titre"])
oldarts.append((line["titre"], line))
elif line["type"] == "section":
oldsects.append(line)
except Exception as e:
print(type(e), e, file=sys.stderr)
print("Incorrect previous text: %s" % previous)
exit()
if previous_step_metas and not previous_step_metas.get("skip"):
try:
assert(previous_step_metas["type"] == "texte")
oldnum = int(find_num.search(previous_step_metas['id']).group(1))
except Exception as e:
print(type(e), e, file=sys.stderr)
print("Incorrect previous step: %s" % previous_step_metas)
exit()
gdoldstep = None
if anteprevious:
gdoldstep = {}
for line in anteprevious:
if line["type"] == "article":
keys = list(line['alineas'].keys())
keys.sort()
gdoldstep[line["titre"]] = [line['alineas'][k] for k in keys]
ALL_ARTICLES = []
def write_json(data):
nonlocal ALL_ARTICLES
ALL_ARTICLES.append(data)
null_reg = re.compile(r'^$')
re_mat_simple = re.compile(r'[IVXDCLM\d]')
re_mat_complex = re.compile(r'L[O.\s]*[IVXDCLM\d]')
re_mat_complex2 = re.compile(r'\d+-\d+')
re_clean_art = re.compile(r'^"?Art\.?\s*', re.I)
make_sta_reg = lambda x: re.compile(r'^("?Art[\s\.]*)?%s\s*(([\.°\-]+\s*)+)' % re_clean_art.sub('', x))
make_end_reg = lambda x, rich: re.compile(r'^%s[IVXDCLM\d\-]+([\-\.\s]+\d*)*((%s|[A-Z])\s*)*(\(|et\s|%s)' % ('("?[LA][LArRtTO\.\s]+)?' if rich else "", bister, x))
re_sect_chg = re.compile(r'^((chap|t)itre|volume|livre|tome|(sous-)?section)\s+[1-9IVXDC]', re.I)
def get_mark_from_last(text, s, l="", sep="", enable_copyall=False, copyall=False):
log("- GET Extract from " + s + " to " + l)
res = []
try:
start = make_sta_reg(s)
except Exception as e:
print('ERROR', type(e), e, s, l, file=sys.stderr)
exit()
rich = re_mat_complex.match(s) or re_mat_complex2.match(s) or not re_mat_simple.match(s)
if l:
last = make_sta_reg(l)
re_end = None
record = False
for n, i in enumerate(text):
matc = start.match(i)
# log(" TEST: " + i[:50])
if re_end and (re_end.match(i) or re_sect_chg.match(i)):
if l:
re_end = make_end_reg(sep, rich)
l = ""
else:
log(" --> END FOUND")
record = False
break
elif matc:
sep = matc.group(2).strip()
log(" --> START FOUND " + sep)
record = True
if l:
re_end = last
else:
re_end = make_end_reg(sep, rich)
elif copyall is True:
record = True
re_end = null_reg
if n == 0:
i = "%s%s %s" % (s, ". -" if re_mat_simple.match(s) else sep, i)
if record:
log(" copy alinea: " + i[:10])
res.append(i)
# retry and get everything as I before II added if not found
if not res:
if not l and enable_copyall and not copyall:
log(" nothing found, grabbing all article now...")
return get_mark_from_last(text, s, l, sep=sep, copyall=True)
print('ERROR: could not retrieve', s, file=sys.stderr)
return False
return res
get_alineas_text = lambda a: clean_text_for_diff([a[k] for k in sorted(a.keys())])
re_clean_et = re.compile(r'(\s*[\&,]\s*|\s+et\s+)+', re.I)
re_suppr = re.compile(r'\W*suppr(ess|im)', re.I)
re_confo = re.compile(r'\W*(conforme|non[\s\-]*modifi)', re.I)
re_confo_with_txt = re.compile(r'\s*\(\s*(conforme|non[\s\-]*modifié)\s*\)\s*([\W]*\w+)', re.I)
re_clean_subsec_space = re.compile(r'^("?[IVX0-9]{1,4}(\s+[a-z]+)?(\s+[A-Z]{1,4})?)\s*([\.°\-]+)\s*([^\s\)])', re.I)
order = 1
cursec = {'id': ''}
done_titre = False
texte = None
for line_i, line in enumerate(current):
if not line or not "type" in line:
sys.stderr.write("JSON badly formatted, missing field type: %s\n" % line)
exit()
if oldnum and 'source_text' in line and oldnum != line['source_text']:
continue
if line["type"] == "echec":
texte["echec"] = True
texte["expose"] = line["texte"]
write_json(texte)
for a in oldjson:
write_json(a)
break
elif line["type"] == "texte":
texte = dict(line)
# check number of sections is the same as the final text
#if texte["definitif"]:
# assert len(oldsects) == len([x for x in current if x['type'] == 'section'])
else:
if not done_titre:
write_json(texte)
done_titre = True
if line["type"] != "article":
if texte['definitif']:
try:
cursec = oldsects.pop(0)
assert(cursec["type_section"] == line["type_section"])
except:
print("ERROR: Problem while renumbering sections: ", line['titre'], " is not ", cursec, '\n', file=sys.stderr)
# exit()
if line["id"] != cursec["id"]:
log("DEBUG: Matched section %s (%s) with old section %s (%s)" % (line["id"], line.get('titre'), cursec["id"], cursec.get('titre')))
line["newid"] = line["id"]
line["id"] = cursec["id"]
write_json(line)
else:
keys = list(line['alineas'].keys())
keys.sort()
alineas = [line['alineas'][k] for k in keys]
mult = line['titre'].split(' à ')
is_mult = (len(mult) > 1)
if is_mult:
st = mult[0].strip()
ed = mult[1].strip()
if re_suppr.match(line['statut']) or (len(alineas) == 1 and re_suppr.match(alineas[0])):
if (st not in oldartids and ed not in oldartids) or (st in oldstatus and re_suppr.match(oldstatus[st]) and ed in oldstatus and re_suppr.match(oldstatus[ed])):
log("DEBUG: SKIP already deleted articles %s to %s" % (st, ed))
continue
log("DEBUG: Marking as deleted articles %s à %s" % (st, ed))
mult_type = "sup"
elif re_confo.match(line['statut']) or (len(alineas) == 1 and re_confo.match(alineas[0])):
log("DEBUG: Recovering art conformes %s à %s" % (st, ed))
mult_type = "conf"
else:
print("ERROR: Found multiple article which I don't know what to do with", line['titre'], line, file=sys.stderr)
exit()
line['titre'] = st
cur = ""
if texte['definitif']:
try:
# recover the old article and mark those deleted as deleted
while True:
_, oldart = oldarts[0]
# first, mark the old articles as deleted via the concordance table
if oldart['titre'].lower() in table_concordance:
new_art = table_concordance[oldart['titre']]
if 'suppr' in new_art:
c, a = oldarts.pop(0)
oldartids.remove(c)
if olddepot:
log("DEBUG: Marking art %s as supprimé (thanks to concordance table)" % c)
a["order"] = order
order += 1
write_json(a)
continue
# as a backup, use the detected status to wait for a non-deleted article
if re_suppr.match(oldart['statut']):
c, a = oldarts.pop(0)
oldartids.remove(c)
if olddepot:
log("DEBUG: Marking art %s as supprimé (recovered)" % c)
a["order"] = order
order += 1
write_json(a)
else:
break
except Exception as e:
print("ERROR: Problem while renumbering articles", line, "\n", oldart, "\n", type(e), e, file=sys.stderr)
exit()
# detect matching errors
if oldart['titre'] in table_concordance:
new_art = table_concordance[oldart['titre']]
if new_art.lower() != line['titre'].lower():
print("ERROR: true concordance is different: when parsing article '%s', we matched it with '%s' which should be matched to '%s' (from concordance table) " % (line['titre'] , oldart['titre'], new_art))
match = None
for oldart_title, newart_title in table_concordance.items():
if newart_title.lower() == line['titre'].lower():
match = oldart_title
print(' -> it should have been matched with article %s' % oldart_title)
break
else:
print(' -> it should have been deleted')
# if article not matching but here in the concordance table, introduce it as a new one
# since it can correspond to an article deleted by the Senate in Nouvelle lecture
# or an article introduced in CMP hémicycle
# /!\ this can only happen during a lecture définitive or a CMP hémicycle
if (step.get('stage') == 'l. définitive' or (
step.get('step') == 'hemicycle' and step.get('stage') == 'CMP')
) and match:
log("DEBUG: Marking art %s as nouveau" % line['titre'])
if "section" in line and cursec['id'] != line["section"]:
line["section"] = cursec["id"]
a = line
a["order"] = order
a["status"] = "nouveau"
if a['titre'] != match:
a['newtitre'] = a['titre']
a['titre'] = match
order += 1
write_json(a)
continue
else:
exit()
log("DEBUG: article '%s' matched with old article '%s'" % (line['titre'], oldart['titre']))
oldtxt = get_alineas_text(oldart["alineas"])
txt = get_alineas_text(line["alineas"])
similarity = compute_similarity(oldtxt, txt)
if similarity < 0.75 and not olddepot and not step.get('stage') == 'constitutionnalité':
print("WARNING BIG DIFFERENCE BETWEEN RENUMBERED ARTICLE", oldart["titre"], "<->", line["titre"], len(oldtxt), "Vs", len(txt), "chars, similarity; %.2f" % similarity, file=sys.stderr)
if line['titre'] != oldart['titre']:
line['newtitre'] = line['titre']
line['titre'] = oldart['titre']
if "section" in line and cursec['id'] != line["section"]:
line["section"] = cursec["id"]
if oldarts:
while oldarts:
cur, a = oldarts.pop(0)
if line['titre'] in oldartids or article_is_lower(cur, line['titre']):
oldartids.remove(cur)
else:
oldarts.insert(0, (cur, a))
break
if cur == line['titre']:
break
if a["statut"].startswith("conforme"):
log("DEBUG: Recovering art conforme %s" % cur)
a["statut"] = "conforme"
a["order"] = order
order += 1
write_json(a)
elif not re_suppr.match(a["statut"]):
# if the last line of text was some dots, it means that we should keep
# the articles as-is if they are not deleted
last_block_was_dots_and_not_an_article = False
for block in reversed(current[:line_i]):
if block['type'] == 'dots':
last_block_was_dots_and_not_an_article = True
break
if block['type'] == 'article':
break
if last_block_was_dots_and_not_an_article:
# ex: https://www.senat.fr/leg/ppl09-304.html
log("DEBUG: Recovering art as non-modifié via dots %s" % cur)
a["statut"] = "non modifié"
a["order"] = order
order += 1
write_json(a)
else:
log("DEBUG: Marking art %s as supprimé because it disappeared" % cur)
a["statut"] = "supprimé"
a["alineas"] = dict()
a["order"] = order
order += 1
write_json(a)
if is_mult:
if ed not in oldartids or cur != line['titre']:
if mult_type == "sup":
print("WARNING: could not find first or last part of multiple article to be removed:", line['titre'], "to", ed, "(last found:", cur+")", file=sys.stderr)
continue
print("ERROR: dealing with multiple article", line['titre'], "to", ed, "Could not find first or last part in last step (last found:", cur+")", file=sys.stderr)
exit()
while True:
if mult_type == "sup" and not re_suppr.match(a["statut"]):
log("DEBUG: Marking art %s as supprimé (mult)" % cur)
a["statut"] = "supprimé"
a["alineas"] = dict()
a["order"] = order
order += 1
write_json(a)
elif mult_type == "conf":
log("DEBUG: Recovering art conforme %s (mult)" % cur)
a["statut"] = "conforme"
a["order"] = order
order += 1
write_json(a)
if cur == ed or not oldarts:
break
cur, a = oldarts.pop(0)
continue
if (re_suppr.match(line["statut"]) or (len(alineas) == 1 and re_suppr.match(alineas[0]))) and (line['titre'] not in oldstatus or re_suppr.match(oldstatus[line['titre']])):
continue
# Clean empty articles with only "Non modifié" and include text from previous step
if alineas and re_confo.match(alineas[0]) and alineas[0].endswith(')'):
if not line['titre'] in oldstep:
sys.stderr.write("WARNING: found repeated article %s missing from previous step: %s (article is ignored)\n" % (line['titre'], line['alineas']))
# ignore empty non-modified
continue
else:
log("DEBUG: get back Art %s" % line['titre'])
alineas = oldstep[line['titre']]
gd_text = []
enable_copyall = True
for j, text in enumerate(alineas):
if "(Non modifi" in text and not line['titre'] in oldstep:
sys.stderr.write("WARNING: found repeated article missing %s from previous step: %s\n" % (line['titre'], text))
elif re_confo_with_txt.search(text):
text = re_confo_with_txt.sub(r' \2', text)
text = re_clean_subsec_space.sub(r'\1\4 \5', text)
gd_text.append(text)
elif "(Non modifi" in text:
part = re.split("\s*([\.°\-]+\s*)+\s*\(Non", text)
if not part:
log("ERROR trying to get non-modifiés")
exit()
pieces = re_clean_et.sub(',', part[0])
log("EXTRACT non-modifiés for " + line['titre'] + ": " + pieces)
piece = []
for todo in pieces.split(','):
# Extract series of non-modified subsections of articles from previous version.
if " à " in todo:
start = re.split(" à ", todo)[0]
end = re.split(" à ", todo)[1]
mark = get_mark_from_last(oldstep[line['titre']], start, end, sep=part[1:], enable_copyall=enable_copyall)
if mark is False and gdoldstep:
mark = get_mark_from_last(gdoldstep[line['titre']], start, end, sep=part[1:], enable_copyall=enable_copyall)
if mark is False:
exit()
enable_copyall = False
piece.extend(mark)
# Extract set of non-modified subsections of articles from previous version.
elif todo:
mark = get_mark_from_last(oldstep[line['titre']], todo, sep=part[1:], enable_copyall=enable_copyall)
if mark is False and gdoldstep:
mark = get_mark_from_last(gdoldstep[line['titre']], todo, sep=part[1:], enable_copyall=enable_copyall)
if mark is False:
exit()
enable_copyall = False
piece.extend(mark)
gd_text.extend(piece)
else:
gd_text.append(text)
line['alineas'] = dict()
line['order'] = order
order += 1
for i, t in enumerate(gd_text):
line['alineas']["%03d" % (i+1)] = t
write_json(line)
if texte['definitif'] and oldsects and oldarts:
print("ERROR: %s sections left:\n%s" % (len(oldsects), oldsects), file=sys.stderr)
#exit()
while oldarts:
cur, a = oldarts.pop(0)
oldartids.remove(cur)
if texte['definitif'] and not re_suppr.match(a["statut"]):
print("ERROR: %s articles left:\n%s %s" % (len(oldarts)+1, cur, oldartids), file=sys.stderr)
exit()
if not texte.get('echec', '') and a["statut"].startswith("conforme"):
log("DEBUG: Recovering art conforme %s (leftovers)" % cur)
a["statut"] = "conforme"
a["order"] = order
order += 1
write_json(a)
# do not keep already deleted articles but mark as deleted missing ones
elif not re_suppr.match(a["statut"]) or texte.get('echec', ''):
# if the last line of text was some dots, it means that we should keep
# the articles as-is if they are not deleted
if line['type'] == 'dots':
# ex: https://www.senat.fr/leg/ppl09-304.html
log("DEBUG: Recovering art as non-modifié via dots %s (leftovers)" % cur)
a["statut"] = "non modifié"
a["order"] = order
order += 1
write_json(a)
else:
log("DEBUG: Marking art %s as supprimé (leftovers)" % cur)
a["statut"] = "supprimé"
a["alineas"] = dict()
a["order"] = order
order += 1
write_json(a)
return ALL_ARTICLES
def _recommendation(self, venues_data: CaseInsensitiveDict,
user_pref: Dict) -> Dict:
logging.info(
f"Generating a recommendation report to advise which venues should the team members go."
)
places_to_avoid = list()
for bad_food, _users in user_pref['bad_food_user_mappings'].items():
for u in _users:
for venue, available_food in venues_data.items():
# Assumption: if the venue offers anything else but what the user won't eat, then
# it is assumed that the user is comfortable in visiting that place. However,
# if the venue only offers the very food that the user won't eat then it is assumed that
# that venue should be avoided...
list_of_foods_offered_by_venue = list(
set(x.strip().lower() for x in available_food['food']))
if ([bad_food] == list_of_foods_offered_by_venue) or (
len(list_of_foods_offered_by_venue) == 0):
# [bad_food] == list_of_foods_offered_by_venue will return true only when the
# venue offers one type of dish and that happens to be the one that the user won't eat.
# len(list_of_foods_offered_by_venue) == 0 means that venue doesn't offer anything at all,
# hence it should be avoided.
places_to_avoid.append({
venue:
f'There is nothing for {u.split()[0]} to eat.'
})
for _user, drinks in user_pref['user_drinks_mappings'].items():
for _v_name, available_drinks in venues_data.items():
set_of_drinks_offered_by_venue = set(
x.strip().lower() for x in available_drinks['drinks'])
if len(
set(drinks).intersection(
set_of_drinks_offered_by_venue)) == 0:
# len(set(drinks).intersection(set_of_drinks_offered_by_venue)) == 0 will return True if
# the venue offers none of the drinks that are desired by the users; In either case
# this suggests that the team cannot go to that venue.
places_to_avoid.append({
_v_name:
f'There is nothing for {_user.split()[0]} to drink.'
})
avoid_dict = self._merge_dicts(places_to_avoid, default_type=list)
return {
'places_to_visit': [
v for v in venues_data.keys()
if v not in [key for key in avoid_dict.keys()]
],
'places_to_avoid': [{
'name': key,
'reason': value
} for key, value in avoid_dict.items()]
}
class Blink:
"""Class to initialize communication."""
def __init__(
self,
refresh_rate=DEFAULT_REFRESH,
motion_interval=DEFAULT_MOTION_INTERVAL,
no_owls=False,
):
"""
Initialize Blink system.
:param refresh_rate: Refresh rate of blink information.
Defaults to 15 (seconds)
:param motion_interval: How far back to register motion in minutes.
Defaults to last refresh time.
Useful for preventing motion_detected property
from de-asserting too quickly.
:param no_owls: Disable searching for owl entries (blink mini cameras only known entity). Prevents an uneccessary API call if you don't have these in your network.
"""
self.auth = Auth()
self.account_id = None
self.client_id = None
self.network_ids = []
self.urls = None
self.sync = CaseInsensitiveDict({})
self.last_refresh = None
self.refresh_rate = refresh_rate
self.networks = []
self.cameras = CaseInsensitiveDict({})
self.video_list = CaseInsensitiveDict({})
self.motion_interval = motion_interval
self.version = __version__
self.available = False
self.key_required = False
self.homescreen = {}
self.no_owls = no_owls
@util.Throttle(seconds=MIN_THROTTLE_TIME)
def refresh(self, force=False, force_cache=False):
"""
Perform a system refresh.
:param force: Used to override throttle, resets refresh
:param force_cache: Used to force update without overriding throttle
"""
if self.check_if_ok_to_update() or force or force_cache:
if not self.available:
self.setup_post_verify()
self.get_homescreen()
for sync_name, sync_module in self.sync.items():
_LOGGER.debug("Attempting refresh of sync %s", sync_name)
sync_module.refresh(force_cache=(force or force_cache))
if not force_cache:
# Prevents rapid clearing of motion detect property
self.last_refresh = int(time.time())
return True
return False
def start(self):
"""Perform full system setup."""
try:
self.auth.startup()
self.setup_login_ids()
self.setup_urls()
self.get_homescreen()
except (LoginError, TokenRefreshFailed, BlinkSetupError):
_LOGGER.error("Cannot setup Blink platform.")
self.available = False
return False
self.key_required = self.auth.check_key_required()
if self.key_required:
if self.auth.no_prompt:
return True
self.setup_prompt_2fa()
return self.setup_post_verify()
def setup_prompt_2fa(self):
"""Prompt for 2FA."""
email = self.auth.data["username"]
pin = input(f"Enter code sent to {email}: ")
result = self.auth.send_auth_key(self, pin)
self.key_required = not result
def setup_post_verify(self):
"""Initialize blink system after verification."""
try:
self.setup_networks()
networks = self.setup_network_ids()
cameras = self.setup_camera_list()
except BlinkSetupError:
self.available = False
return False
for name, network_id in networks.items():
sync_cameras = cameras.get(network_id, {})
self.setup_sync_module(name, network_id, sync_cameras)
self.cameras = self.merge_cameras()
self.available = True
self.key_required = False
return True
def setup_sync_module(self, name, network_id, cameras):
"""Initialize a sync module."""
self.sync[name] = BlinkSyncModule(self, name, network_id, cameras)
self.sync[name].start()
def get_homescreen(self):
"""Get homecreen information."""
if self.no_owls:
_LOGGER.debug("Skipping owl extraction.")
self.homescreen = {}
return
self.homescreen = api.request_homescreen(self)
def setup_owls(self):
"""Check for mini cameras."""
network_list = []
camera_list = []
try:
for owl in self.homescreen["owls"]:
name = owl["name"]
network_id = str(owl["network_id"])
if network_id in self.network_ids:
camera_list.append(
{network_id: {"name": name, "id": network_id, "type": "mini"}}
)
continue
if owl["onboarded"]:
network_list.append(str(network_id))
self.sync[name] = BlinkOwl(self, name, network_id, owl)
self.sync[name].start()
except KeyError:
# No sync-less devices found
pass
self.network_ids.extend(network_list)
return camera_list
def setup_camera_list(self):
"""Create camera list for onboarded networks."""
all_cameras = {}
response = api.request_camera_usage(self)
try:
for network in response["networks"]:
camera_network = str(network["network_id"])
if camera_network not in all_cameras:
all_cameras[camera_network] = []
for camera in network["cameras"]:
all_cameras[camera_network].append(
{"name": camera["name"], "id": camera["id"]}
)
mini_cameras = self.setup_owls()
for camera in mini_cameras:
for network, camera_info in camera.items():
all_cameras[network].append(camera_info)
return all_cameras
except (KeyError, TypeError):
_LOGGER.error("Unable to retrieve cameras from response %s", response)
raise BlinkSetupError
def setup_login_ids(self):
"""Retrieve login id numbers from login response."""
self.client_id = self.auth.client_id
self.account_id = self.auth.account_id
def setup_urls(self):
"""Create urls for api."""
try:
self.urls = util.BlinkURLHandler(self.auth.region_id)
except TypeError:
_LOGGER.error(
"Unable to extract region is from response %s", self.auth.login_response
)
raise BlinkSetupError
def setup_networks(self):
"""Get network information."""
response = api.request_networks(self)
try:
self.networks = response["summary"]
except (KeyError, TypeError):
raise BlinkSetupError
def setup_network_ids(self):
"""Create the network ids for onboarded networks."""
all_networks = []
network_dict = {}
try:
for network, status in self.networks.items():
if status["onboarded"]:
all_networks.append(f"{network}")
network_dict[status["name"]] = network
except AttributeError:
_LOGGER.error(
"Unable to retrieve network information from %s", self.networks
)
raise BlinkSetupError
self.network_ids = all_networks
return network_dict
def check_if_ok_to_update(self):
"""Check if it is ok to perform an http request."""
current_time = int(time.time())
last_refresh = self.last_refresh
if last_refresh is None:
last_refresh = 0
if current_time >= (last_refresh + self.refresh_rate):
return True
return False
def merge_cameras(self):
"""Merge all sync camera dicts into one."""
combined = CaseInsensitiveDict({})
for sync in self.sync:
combined = util.merge_dicts(combined, self.sync[sync].cameras)
return combined
def save(self, file_name):
"""Save login data to file."""
util.json_save(self.auth.login_attributes, file_name)
def download_videos(
self, path, since=None, camera="all", stop=10, delay=1, debug=False
):
"""
Download all videos from server since specified time.
:param path: Path to write files. /path/<cameraname>_<recorddate>.mp4
:param since: Date and time to get videos from.
Ex: "2018/07/28 12:33:00" to retrieve videos since
July 28th 2018 at 12:33:00
:param camera: Camera name to retrieve. Defaults to "all".
Use a list for multiple cameras.
:param stop: Page to stop on (~25 items per page. Default page 10).
:param delay: Number of seconds to wait in between subsequent video downloads.
:param debug: Set to TRUE to prevent downloading of items.
Instead of downloading, entries will be printed to log.
"""
if since is None:
since_epochs = self.last_refresh
else:
parsed_datetime = parse(since, fuzzy=True)
since_epochs = parsed_datetime.timestamp()
formatted_date = util.get_time(time_to_convert=since_epochs)
_LOGGER.info("Retrieving videos since %s", formatted_date)
if not isinstance(camera, list):
camera = [camera]
for page in range(1, stop):
response = api.request_videos(self, time=since_epochs, page=page)
_LOGGER.debug("Processing page %s", page)
try:
result = response["media"]
if not result:
raise KeyError
except (KeyError, TypeError):
_LOGGER.info("No videos found on page %s. Exiting.", page)
break
self._parse_downloaded_items(result, camera, path, delay, debug)
def _parse_downloaded_items(self, result, camera, path, delay, debug):
"""Parse downloaded videos."""
for item in result:
try:
created_at = item["created_at"]
camera_name = item["device_name"]
is_deleted = item["deleted"]
address = item["media"]
except KeyError:
_LOGGER.info("Missing clip information, skipping...")
continue
if camera_name not in camera and "all" not in camera:
_LOGGER.debug("Skipping videos for %s.", camera_name)
continue
if is_deleted:
_LOGGER.debug("%s: %s is marked as deleted.", camera_name, address)
continue
clip_address = f"{self.urls.base_url}{address}"
filename = f"{camera_name}-{created_at}"
filename = f"{slugify(filename)}.mp4"
filename = os.path.join(path, filename)
if not debug:
if os.path.isfile(filename):
_LOGGER.info("%s already exists, skipping...", filename)
continue
response = api.http_get(
self,
url=clip_address,
stream=True,
json=False,
timeout=TIMEOUT_MEDIA,
)
with open(filename, "wb") as vidfile:
copyfileobj(response.raw, vidfile)
_LOGGER.info("Downloaded video to %s", filename)
else:
print(
(
f"Camera: {camera_name}, Timestamp: {created_at}, "
"Address: {address}, Filename: {filename}"
)
)
if delay > 0:
time.sleep(delay)
def complete(current, previous, step, table_concordance, anteprevious=None):
current = copy.deepcopy(current)
previous = copy.deepcopy(previous)
table_concordance = CaseInsensitiveDict(table_concordance)
DEBUG = '--debug' in sys.argv
def log(text):
if DEBUG:
print(text, file=sys.stderr)
def exit():
raise Exception('[complete_articles] Fatal error')
find_num = re.compile(r'-[a-z]*(\d+)\D?$')
oldnum = 0
oldstep = {}
oldjson = []
oldstatus = {}
oldartids = []
oldarts = []
oldsects = []
try:
for line in previous:
if line["type"] != "texte":
oldjson.append(line)
else:
oldnum = int(find_num.search(line['id']).group(1))
olddepot = line['depot']
if line["type"] == "article":
keys = list(line['alineas'].keys())
keys.sort()
oldstep[line["titre"]] = [line['alineas'][k] for k in keys]
oldstatus[line["titre"]] = line['statut']
oldartids.append(line["titre"])
oldarts.append((line["titre"], line))
elif line["type"] == "section":
oldsects.append(line)
except Exception as e:
print(type(e), e, file=sys.stderr)
log("No previous step found at %s" % sys.argv[2])
exit()
gdoldstep = None
if anteprevious:
gdoldstep = {}
for line in anteprevious:
if line["type"] == "article":
keys = list(line['alineas'].keys())
keys.sort()
gdoldstep[line["titre"]] = [line['alineas'][k] for k in keys]
ALL_ARTICLES = []
def write_json(data):
nonlocal ALL_ARTICLES
ALL_ARTICLES.append(data)
null_reg = re.compile(r'^$')
re_mat_simple = re.compile(r'[IVXDCLM\d]')
re_mat_complex = re.compile(r'L[O.\s]*[IVXDCLM\d]')
re_mat_complex2 = re.compile(r'\d+-\d+')
re_clean_art = re.compile(r'^"?Art\.?\s*', re.I)
make_sta_reg = lambda x: re.compile(
r'^("?Art[\s\.]*)?%s\s*(([\.°\-]+\s*)+)' % re_clean_art.sub('', x))
make_end_reg = lambda x, rich: re.compile(
r'^%s[IVXDCLM\d\-]+([\-\.\s]+\d*)*((%s|[A-Z])\s*)*(\(|et\s|%s)' %
('("?[LA][LArRtTO\.\s]+)?' if rich else "", bister, x))
re_sect_chg = re.compile(
r'^((chap|t)itre|volume|livre|tome|(sous-)?section)\s+[1-9IVXDC]',
re.I)
def get_mark_from_last(text,
s,
l="",
sep="",
enable_copyall=False,
copyall=False):
log("- GET Extract from " + s + " to " + l)
res = []
try:
start = make_sta_reg(s)
except Exception as e:
print('ERROR', type(e), e, s, l, file=sys.stderr)
exit()
rich = re_mat_complex.match(s) or re_mat_complex2.match(
s) or not re_mat_simple.match(s)
if l:
last = make_sta_reg(l)
re_end = None
record = False
for n, i in enumerate(text):
matc = start.match(i)
# log(" TEST: " + i[:50])
if re_end and (re_end.match(i) or re_sect_chg.match(i)):
if l:
re_end = make_end_reg(sep, rich)
l = ""
else:
log(" --> END FOUND")
record = False
break
elif matc:
sep = matc.group(2).strip()
log(" --> START FOUND " + sep)
record = True
if l:
re_end = last
else:
re_end = make_end_reg(sep, rich)
elif copyall is True:
record = True
re_end = null_reg
if n == 0:
i = "%s%s %s" % (s, ". -"
if re_mat_simple.match(s) else sep, i)
if record:
log(" copy alinea")
res.append(i)
# retry and get everything as I before II added if not found
if not res:
if not l and enable_copyall and not copyall:
log(" nothing found, grabbing all article now...")
return get_mark_from_last(text, s, l, sep=sep, copyall=True)
print('ERROR: could not retrieve', s, file=sys.stderr)
return False
return res
get_alineas_text = lambda a: clean_text_for_diff(
[a[k] for k in sorted(a.keys())])
re_clean_et = re.compile(r'(\s*[\&,]\s*|\s+et\s+)+', re.I)
re_suppr = re.compile(r'\W*suppr(ess|im)', re.I)
re_confo = re.compile(r'\W*(conforme|non[\s\-]*modifi)', re.I)
re_confo_with_txt = re.compile(
r'\s*\(\s*(conforme|non[\s\-]*modifié)\s*\)\s*([\W]*\w+)', re.I)
re_clean_subsec_space = re.compile(
r'^("?[IVX0-9]{1,4}(\s+[a-z]+)?(\s+[A-Z]{1,4})?)\s*([\.°\-]+)\s*([^\s\)])',
re.I)
order = 1
cursec = {'id': ''}
done_titre = False
texte = None
for line_i, line in enumerate(current):
if not line or not "type" in line:
sys.stderr.write("JSON badly formatted, missing field type: %s\n" %
line)
exit()
if oldnum and 'source_text' in line and oldnum != line['source_text']:
continue
if line["type"] == "echec":
texte["echec"] = True
texte["expose"] = line["texte"]
write_json(texte)
for a in oldjson:
write_json(a)
break
elif line["type"] == "texte":
texte = dict(line)
# check number of sections is the same as the final text
#if texte["definitif"]:
# assert len(oldsects) == len([x for x in current if x['type'] == 'section'])
else:
if not done_titre:
write_json(texte)
done_titre = True
if line["type"] != "article":
if texte['definitif']:
try:
cursec = oldsects.pop(0)
assert (cursec["type_section"] == line["type_section"])
except:
print("ERROR: Problem while renumbering sections: ",
line['titre'],
" is not ",
cursec,
'\n',
file=sys.stderr)
# exit()
if line["id"] != cursec["id"]:
log("DEBUG: Matched section %s (%s) with old section %s (%s)"
% (line["id"], line.get('titre'), cursec["id"],
cursec.get('titre')))
line["newid"] = line["id"]
line["id"] = cursec["id"]
write_json(line)
else:
keys = list(line['alineas'].keys())
keys.sort()
alineas = [line['alineas'][k] for k in keys]
mult = line['titre'].split(' à ')
is_mult = (len(mult) > 1)
if is_mult:
st = mult[0].strip()
ed = mult[1].strip()
if re_suppr.match(line['statut']) or (
len(alineas) == 1 and re_suppr.match(alineas[0])):
if (st not in oldartids and ed not in oldartids) or (
st in oldstatus and re_suppr.match(
oldstatus[st]) and ed in oldstatus
and re_suppr.match(oldstatus[ed])):
log("DEBUG: SKIP already deleted articles %s to %s"
% (st, ed))
continue
log("DEBUG: Marking as deleted articles %s à %s" %
(st, ed))
mult_type = "sup"
elif re_confo.match(line['statut']) or (
len(alineas) == 1 and re_confo.match(alineas[0])):
log("DEBUG: Recovering art conformes %s à %s" %
(st, ed))
mult_type = "conf"
else:
print(
"ERROR: Found multiple article which I don't know what to do with",
line['titre'],
line,
file=sys.stderr)
exit()
line['titre'] = st
cur = ""
if texte['definitif']:
try:
# recover the old article and mark those deleted as deleted
while True:
_, oldart = oldarts[0]
# first, mark the old articles as deleted via the concordance table
if oldart['titre'].lower() in table_concordance:
new_art = table_concordance[oldart['titre']]
if 'suppr' in new_art:
c, a = oldarts.pop(0)
oldartids.remove(c)
if olddepot:
log("DEBUG: Marking art %s as supprimé (thanks to concordance table)"
% c)
a["order"] = order
order += 1
write_json(a)
continue
# as a backup, use the detected status to wait for a non-deleted article
if re_suppr.match(oldart['statut']):
c, a = oldarts.pop(0)
oldartids.remove(c)
if olddepot:
log("DEBUG: Marking art %s as supprimé (recovered)"
% c)
a["order"] = order
order += 1
write_json(a)
else:
break
except:
print("ERROR: Problem while renumbering articles",
line,
"\n",
oldart,
file=sys.stderr)
exit()
# detect matching errors
if oldart['titre'] in table_concordance:
new_art = table_concordance[oldart['titre']]
if new_art.lower() != line['titre'].lower():
print(
"ERROR: true concordance is different: when parsing article '%s', we matched it with '%s' which should be matched to '%s' (from concordance table) "
% (line['titre'], oldart['titre'], new_art))
match = None
for oldart_title, newart_title in table_concordance.items(
):
if newart_title.lower() == line['titre'].lower(
):
match = oldart_title
print(
' -> it should have been matched with article %s'
% oldart_title)
break
else:
print(' -> it should have been deleted')
# if article not matching but here in the concordance table, introduce it as a new one
# since it can correspond to an article deleted by the Senate in Nouvelle lecture
# or an article introduced in CMP hémicycle
# /!\ this can only happen during a lecture définitive or a CMP hémicycle
if (step.get('stage') == 'l. définitive' or
(step.get('step') == 'hemicycle'
and step.get('stage') == 'CMP')) and match:
log("DEBUG: Marking art %s as nouveau" %
line['titre'])
if "section" in line and cursec['id'] != line[
"section"]:
line["section"] = cursec["id"]
a = line
a["order"] = order
a["status"] = "nouveau"
if a['titre'] != match:
a['newtitre'] = a['titre']
a['titre'] = match
order += 1
write_json(a)
continue
else:
exit()
log("DEBUG: article '%s' matched with old article '%s'" %
(line['titre'], oldart['titre']))
oldtxt = get_alineas_text(oldart["alineas"])
txt = get_alineas_text(line["alineas"])
similarity = compute_similarity(oldtxt, txt)
if similarity < 0.75 and not olddepot and not step.get(
'stage') == 'constitutionnalité':
print(
"WARNING BIG DIFFERENCE BETWEEN RENUMBERED ARTICLE",
oldart["titre"],
"<->",
line["titre"],
len(oldtxt),
"Vs",
len(txt),
"chars, similarity; %.2f" % similarity,
file=sys.stderr)
if line['titre'] != oldart['titre']:
line['newtitre'] = line['titre']
line['titre'] = oldart['titre']
if "section" in line and cursec['id'] != line["section"]:
line["section"] = cursec["id"]
if oldarts:
while oldarts:
cur, a = oldarts.pop(0)
if line['titre'] in oldartids or article_is_lower(
cur, line['titre']):
oldartids.remove(cur)
else:
oldarts.insert(0, (cur, a))
break
if cur == line['titre']:
break
if a["statut"].startswith("conforme"):
log("DEBUG: Recovering art conforme %s" % cur)
a["statut"] = "conforme"
a["order"] = order
order += 1
write_json(a)
elif not re_suppr.match(a["statut"]):
# if the last line of text was some dots, it means that we should keep
# the articles as-is if they are not deleted
last_block_was_dots_and_not_an_article = False
for block in reversed(current[:line_i]):
if block['type'] == 'dots':
last_block_was_dots_and_not_an_article = True
break
if block['type'] == 'article':
break
if last_block_was_dots_and_not_an_article:
# ex: https://www.senat.fr/leg/ppl09-304.html
log("DEBUG: Recovering art as non-modifié via dots %s"
% cur)
a["statut"] = "non modifié"
a["order"] = order
order += 1
write_json(a)
else:
log("DEBUG: Marking art %s as supprimé because it disappeared"
% cur)
a["statut"] = "supprimé"
a["alineas"] = dict()
a["order"] = order
order += 1
write_json(a)
if is_mult:
if ed not in oldartids or cur != line['titre']:
if mult_type == "sup":
print(
"WARNING: could not find first or last part of multiple article to be removed:",
line['titre'],
"to",
ed,
"(last found:",
cur + ")",
file=sys.stderr)
continue
print(
"ERROR: dealing with multiple article",
line['titre'],
"to",
ed,
"Could not find first or last part in last step (last found:",
cur + ")",
file=sys.stderr)
exit()
while True:
if mult_type == "sup" and not re_suppr.match(
a["statut"]):
log("DEBUG: Marking art %s as supprimé (mult)" %
cur)
a["statut"] = "supprimé"
a["alineas"] = dict()
a["order"] = order
order += 1
write_json(a)
elif mult_type == "conf":
log("DEBUG: Recovering art conforme %s (mult)" %
cur)
a["statut"] = "conforme"
a["order"] = order
order += 1
write_json(a)
if cur == ed or not oldarts:
break
cur, a = oldarts.pop(0)
continue
if (re_suppr.match(line["statut"]) or
(len(alineas) == 1 and re_suppr.match(alineas[0]))) and (
line['titre'] not in oldstatus
or re_suppr.match(oldstatus[line['titre']])):
continue
# Clean empty articles with only "Non modifié" and include text from previous step
if alineas and re_confo.match(
alineas[0]) and alineas[0].endswith(')'):
if not line['titre'] in oldstep:
sys.stderr.write(
"WARNING: found repeated article %s missing from previous step: %s (article is ignored)\n"
% (line['titre'], line['alineas']))
# ignore empty non-modified
continue
else:
log("DEBUG: get back Art %s" % line['titre'])
alineas = oldstep[line['titre']]
gd_text = []
enable_copyall = True
for j, text in enumerate(alineas):
if "(Non modifi" in text and not line['titre'] in oldstep:
sys.stderr.write(
"WARNING: found repeated article missing %s from previous step: %s\n"
% (line['titre'], text))
elif re_confo_with_txt.search(text):
text = re_confo_with_txt.sub(r' \2', text)
text = re_clean_subsec_space.sub(r'\1\4 \5', text)
gd_text.append(text)
elif "(Non modifi" in text:
part = re.split("\s*([\.°\-]+\s*)+\s*\(Non", text)
if not part:
log("ERROR trying to get non-modifiés")
exit()
pieces = re_clean_et.sub(',', part[0])
log("EXTRACT non-modifiés for " + line['titre'] +
": " + pieces)
piece = []
for todo in pieces.split(','):
# Extract series of non-modified subsections of articles from previous version.
if " à " in todo:
start = re.split(" à ", todo)[0]
end = re.split(" à ", todo)[1]
mark = get_mark_from_last(
oldstep[line['titre']],
start,
end,
sep=part[1:],
enable_copyall=enable_copyall)
if mark is False and gdoldstep:
mark = get_mark_from_last(
gdoldstep[line['titre']],
start,
end,
sep=part[1:],
enable_copyall=enable_copyall)
if mark is False:
exit()
enable_copyall = False
piece.extend(mark)
# Extract set of non-modified subsections of articles from previous version.
elif todo:
mark = get_mark_from_last(
oldstep[line['titre']],
todo,
sep=part[1:],
enable_copyall=enable_copyall)
if mark is False and gdoldstep:
mark = get_mark_from_last(
gdoldstep[line['titre']],
todo,
sep=part[1:],
enable_copyall=enable_copyall)
if mark is False:
exit()
enable_copyall = False
piece.extend(mark)
gd_text.extend(piece)
else:
gd_text.append(text)
line['alineas'] = dict()
line['order'] = order
order += 1
for i, t in enumerate(gd_text):
line['alineas']["%03d" % (i + 1)] = t
write_json(line)
if texte['definitif'] and oldsects and oldarts:
print("ERROR: %s sections left:\n%s" % (len(oldsects), oldsects),
file=sys.stderr)
#exit()
while oldarts:
cur, a = oldarts.pop(0)
oldartids.remove(cur)
if texte['definitif'] and not re_suppr.match(a["statut"]):
print("ERROR: %s articles left:\n%s %s" %
(len(oldarts) + 1, cur, oldartids),
file=sys.stderr)
exit()
if not texte.get('echec', '') and a["statut"].startswith("conforme"):
log("DEBUG: Recovering art conforme %s (leftovers)" % cur)
a["statut"] = "conforme"
a["order"] = order
order += 1
write_json(a)
# do not keep already deleted articles but mark as deleted missing ones
elif not re_suppr.match(a["statut"]) or texte.get('echec', ''):
# if the last line of text was some dots, it means that we should keep
# the articles as-is if they are not deleted
if line['type'] == 'dots':
# ex: https://www.senat.fr/leg/ppl09-304.html
log("DEBUG: Recovering art as non-modifié via dots %s (leftovers)"
% cur)
a["statut"] = "non modifié"
a["order"] = order
order += 1
write_json(a)
else:
log("DEBUG: Marking art %s as supprimé (leftovers)" % cur)
a["statut"] = "supprimé"
a["alineas"] = dict()
a["order"] = order
order += 1
write_json(a)
return ALL_ARTICLES
class MockRestURI(object):
"""Representation of a mock URI."""
def __init__(self, uri_dict):
self._uri_dict = uri_dict
self.name = uri_dict.get('name', None)
self.fullpath = uri_dict['request']['path']
self._regexp = False
self.path = urlparse.urlparse(self.fullpath).path
self.querystring = urlparse.parse_qs(
urlparse.urlparse(self.fullpath).query,
keep_blank_values=True
)
self.method = uri_dict['request'].get('method', None)
self.headers = CaseInsensitiveDict(uri_dict['request'].get('headers', {}))
self.return_code = int(uri_dict['response'].get('code', '200'))
self.request_content_type = self.headers.get('Content-Type', '')
self.response_location = uri_dict['response'].get('location', None)
self.response_content = uri_dict['response'].get('content', "")
self.wait = float(uri_dict['response'].get('wait', 0.0))
self.request_data = uri_dict['request'].get('data', None)
self.encoding = uri_dict['request'].get("encoding", None)
self.response_headers = uri_dict['response'].get("headers", {})
def match(self, request):
"""Does this URI match the request?"""
# Match HTTP verb - GET, POST, PUT, DELETE, etc.
if self.method is not None:
if request.command.lower() != self.method.lower():
return False
# Match path
if self.path != urlparse.urlparse(request.path).path:
return False
# Match querystring
if request.querystring() != self.querystring:
return False
# Match headers
if self.headers is not None:
for header_var, header_value in self.headers.items():
request_headers = CaseInsensitiveDict(request.headers)
if request_headers.get(header_var) is None:
return False
req_maintext, req_pdict = cgi.parse_header(request_headers.get(header_var))
mock_maintext, mock_pdict = cgi.parse_header(header_value)
if "boundary" in req_pdict and "boundary" in mock_pdict:
req_pdict['boundary'] = "xxx"
mock_pdict['boundary'] = "xxx"
if req_maintext != mock_maintext:
return False
if mock_pdict != {}:
if req_pdict != mock_pdict:
return False
# Match processed request data
if self.request_data is not None:
# Check if exact match before parsing
if request.body != self.request_data:
if self.request_content_type.startswith("application/json"):
if json.loads(request.body) != json.loads(self.request_data):
return False
elif self.request_content_type.startswith("application/x-www-form-urlencoded"):
if parse_qsl_as_dict(request.body) != parse_qsl_as_dict(self.request_data):
return False
elif self.request_content_type.startswith('application/xml'):
actual_request_data_root = etree.fromstring(request.body)
mock_request_data_root = etree.fromstring(self.request_data)
if not xml_elements_equal(actual_request_data_root, mock_request_data_root):
return False
elif self.request_content_type.startswith("multipart/form-data"):
ctype, pdict = cgi.parse_header(request.headers.get('Content-Type'))
req_multipart = cgi.parse_multipart(
io.BytesIO(request.body.encode('utf8')),
{x: y.encode('utf8') for x, y in pdict.items()}
)
ctype, pdict = cgi.parse_header(self.headers.get('Content-Type'))
mock_multipart = cgi.parse_multipart(
io.BytesIO(self.request_data.encode('utf8')),
{x: y.encode('utf8') for x, y in pdict.items()}
)
return mock_multipart == req_multipart
else:
if request.body != self.request_data:
return False
return True
def querystring_string(self):
# TODO : Refactor this and docstring.
query = ''
for key in self.querystring.keys():
for item in self.querystring[key]:
query += str(key) + '=' + item + "&"
query = query.rstrip("&")
return "?" + query if query else ""
def example_path(self):
return xeger.xeger(self.path) if self._regexp else self.path + self.querystring_string()
def return_code_description(self):
return status_codes.CODES.get(self.return_code)[0]
def request_data_values(self):
if self.request_data is not None:
return self.request_data.get('values', {}).iteritems()
else:
return []
def request_data_type(self):
if self.request_data is not None:
return self.request_data.get('encoding')
else:
return None
class Blink():
"""Class to initialize communication."""
def __init__(self,
username=None,
password=None,
refresh_rate=DEFAULT_REFRESH,
motion_interval=DEFAULT_MOTION_INTERVAL,
legacy_subdomain=False):
"""
Initialize Blink system.
:param username: Blink username (usually email address)
:param password: Blink password
:param refresh_rate: Refresh rate of blink information.
Defaults to 15 (seconds)
:param motion_interval: How far back to register motion in minutes.
Defaults to last refresh time.
Useful for preventing motion_detected property
from de-asserting too quickly.
:param legacy_subdomain: Set to TRUE to use old 'rest.region'
endpoints (only use if you are having
api issues).
"""
self._username = username
self._password = password
self._token = None
self._auth_header = None
self._host = None
self.account_id = None
self.network_ids = []
self.urls = None
self.sync = CaseInsensitiveDict({})
self.region = None
self.region_id = None
self.last_refresh = None
self.refresh_rate = refresh_rate
self.session = create_session()
self.networks = []
self.cameras = CaseInsensitiveDict({})
self.video_list = CaseInsensitiveDict({})
self._login_url = LOGIN_URL
self.login_urls = []
self.motion_interval = motion_interval
self.version = __version__
self.legacy = legacy_subdomain
@property
def auth_header(self):
"""Return the authentication header."""
return self._auth_header
def start(self):
"""
Perform full system setup.
Method logs in and sets auth token, urls, and ids for future requests.
Essentially this is just a wrapper function for ease of use.
"""
if self._username is None or self._password is None:
if not self.login():
return
elif not self.get_auth_token():
return
camera_list = self.get_cameras()
networks = self.get_ids()
for network_name, network_id in networks.items():
if network_id not in camera_list.keys():
camera_list[network_id] = {}
_LOGGER.warning("No cameras found for %s", network_name)
sync_module = BlinkSyncModule(self, network_name, network_id,
camera_list[network_id])
sync_module.start()
self.sync[network_name] = sync_module
self.cameras = self.merge_cameras()
def login(self):
"""Prompt user for username and password."""
self._username = input("Username:"******"Password:"******"Login successful!")
return True
_LOGGER.warning("Unable to login with %s.", self._username)
return False
def get_auth_token(self, is_retry=False):
"""Retrieve the authentication token from Blink."""
if not isinstance(self._username, str):
raise BlinkAuthenticationException(ERROR.USERNAME)
if not isinstance(self._password, str):
raise BlinkAuthenticationException(ERROR.PASSWORD)
self.login_urls = [LOGIN_URL, OLD_LOGIN_URL, LOGIN_BACKUP_URL]
response = self.login_request(is_retry=is_retry)
if not response:
return False
self._host = "{}.{}".format(self.region_id, BLINK_URL)
self._token = response['authtoken']['authtoken']
self.networks = response['networks']
self._auth_header = {'Host': self._host, 'TOKEN_AUTH': self._token}
self.urls = BlinkURLHandler(self.region_id, legacy=self.legacy)
return self._auth_header
def login_request(self, is_retry=False):
"""Make a login request."""
try:
login_url = self.login_urls.pop(0)
except IndexError:
_LOGGER.error("Could not login to blink servers.")
return False
_LOGGER.info("Attempting login with %s", login_url)
response = api.request_login(self,
login_url,
self._username,
self._password,
is_retry=is_retry)
try:
if response.status_code != 200:
response = self.login_request(is_retry=True)
response = response.json()
(self.region_id, self.region), = response['region'].items()
except AttributeError:
_LOGGER.error("Login API endpoint failed with response %s",
response)
return False
except KeyError:
_LOGGER.warning("Could not extract region info.")
self.region_id = 'piri'
self.region = 'UNKNOWN'
self._login_url = login_url
return response
def get_ids(self):
"""Set the network ID and Account ID."""
response = api.request_networks(self)
all_networks = []
network_dict = {}
for network, status in self.networks.items():
if status['onboarded']:
all_networks.append('{}'.format(network))
network_dict[status['name']] = network
# For the first onboarded network we find, grab the account id
for resp in response['networks']:
if str(resp['id']) in all_networks:
self.account_id = resp['account_id']
break
self.network_ids = all_networks
return network_dict
def get_cameras(self):
"""Retrieve a camera list for each onboarded network."""
response = api.request_homescreen(self)
try:
all_cameras = {}
for camera in response['cameras']:
camera_network = str(camera['network_id'])
camera_name = camera['name']
camera_id = camera['id']
camera_info = {'name': camera_name, 'id': camera_id}
if camera_network not in all_cameras:
all_cameras[camera_network] = []
all_cameras[camera_network].append(camera_info)
return all_cameras
except KeyError:
_LOGGER.error("Initialization failue. Could not retrieve cameras.")
return {}
@Throttle(seconds=MIN_THROTTLE_TIME)
def refresh(self, force_cache=False):
"""
Perform a system refresh.
:param force_cache: Force an update of the camera cache
"""
if self.check_if_ok_to_update() or force_cache:
for sync_name, sync_module in self.sync.items():
_LOGGER.debug("Attempting refresh of sync %s", sync_name)
sync_module.refresh(force_cache=force_cache)
if not force_cache:
# Prevents rapid clearing of motion detect property
self.last_refresh = int(time.time())
return True
return False
def check_if_ok_to_update(self):
"""Check if it is ok to perform an http request."""
current_time = int(time.time())
last_refresh = self.last_refresh
if last_refresh is None:
last_refresh = 0
if current_time >= (last_refresh + self.refresh_rate):
return True
return False
def merge_cameras(self):
"""Merge all sync camera dicts into one."""
combined = CaseInsensitiveDict({})
for sync in self.sync:
combined = merge_dicts(combined, self.sync[sync].cameras)
return combined
def download_videos(self,
path,
since=None,
camera='all',
stop=10,
debug=False):
"""
Download all videos from server since specified time.
:param path: Path to write files. /path/<cameraname>_<recorddate>.mp4
:param since: Date and time to get videos from.
Ex: "2018/07/28 12:33:00" to retrieve videos since
July 28th 2018 at 12:33:00
:param camera: Camera name to retrieve. Defaults to "all".
Use a list for multiple cameras.
:param stop: Page to stop on (~25 items per page. Default page 10).
:param debug: Set to TRUE to prevent downloading of items.
Instead of downloading, entries will be printed to log.
"""
if since is None:
since_epochs = self.last_refresh
else:
parsed_datetime = parse(since, fuzzy=True)
since_epochs = parsed_datetime.timestamp()
formatted_date = get_time(time_to_convert=since_epochs)
_LOGGER.info("Retrieving videos since %s", formatted_date)
if not isinstance(camera, list):
camera = [camera]
for page in range(1, stop):
response = api.request_videos(self, time=since_epochs, page=page)
_LOGGER.debug("Processing page %s", page)
try:
result = response['media']
if not result:
raise IndexError
except (KeyError, IndexError):
_LOGGER.info("No videos found on page %s. Exiting.", page)
break
self._parse_downloaded_items(result, camera, path, debug)
def _parse_downloaded_items(self, result, camera, path, debug):
"""Parse downloaded videos."""
for item in result:
try:
created_at = item['created_at']
camera_name = item['device_name']
is_deleted = item['deleted']
address = item['media']
except KeyError:
_LOGGER.info("Missing clip information, skipping...")
continue
if camera_name not in camera and 'all' not in camera:
_LOGGER.debug("Skipping videos for %s.", camera_name)
continue
if is_deleted:
_LOGGER.debug("%s: %s is marked as deleted.", camera_name,
address)
continue
clip_address = "{}{}".format(self.urls.base_url, address)
filename = "{}-{}".format(camera_name, created_at)
filename = "{}.mp4".format(slugify(filename))
filename = os.path.join(path, filename)
if not debug:
if os.path.isfile(filename):
_LOGGER.info("%s already exists, skipping...", filename)
continue
response = api.http_get(self,
url=clip_address,
stream=True,
json=False)
with open(filename, 'wb') as vidfile:
copyfileobj(response.raw, vidfile)
_LOGGER.info("Downloaded video to %s", filename)
else:
print(("Camera: {}, Timestamp: {}, "
"Address: {}, Filename: {}").format(
camera_name, created_at, address, filename))
def _decode_headers(headers: CaseInsensitiveDict,
encoding: str) -> CaseInsensitiveDict:
return CaseInsensitiveDict(
{k.decode(encoding): v.decode(encoding)
for k, v in headers.items()})
class App:
app_config: AppConfig = None
"""
The contents of asyncy.yaml.
"""
config: Config = None
"""
The runtime config for this app.
"""
def __init__(self, app_data: AppData):
self._subscriptions = {}
self.app_id = app_data.app_id
self.app_name = app_data.app_name
self.app_dns = app_data.app_dns
self.config = app_data.config
self.app_config = app_data.app_config
self.version = app_data.version
self.logger = app_data.logger
self.owner_uuid = app_data.owner_uuid
self.owner_email = app_data.owner_email
self.environment = app_data.environment
if app_data.environment is None:
self.environment = {}
else:
self.environment = app_data.environment
self.environment = CaseInsensitiveDict(data=self.environment)
self.stories = app_data.stories['stories']
self.entrypoint = app_data.stories['entrypoint']
self.services = app_data.services
secrets = CaseInsensitiveDict()
for k, v in self.environment.items():
if not isinstance(v, dict):
secrets[k] = v
self.app_context = {
'secrets': secrets,
'hostname': f'{self.app_dns}.{self.config.APP_DOMAIN}',
'version': self.version
}
async def bootstrap(self):
"""
Executes all stories found in stories.json.
This enables the story to listen to pub/sub,
register with the gateway, and queue cron jobs.
"""
await self.start_services()
await self.expose_services()
await self.run_stories()
async def expose_services(self):
for expose in self.app_config.get_expose_config():
await self._expose_service(expose)
async def _expose_service(self, e: Expose):
self.logger.info(f'Exposing service {e.service}/'
f'{e.service_expose_name} '
f'on {e.http_path}')
conf = Dict.find(
self.services, f'{e.service}'
f'.{ServiceConstants.config}'
f'.expose.{e.service_expose_name}')
if conf is None:
raise StoryscriptError(
message=f'Configuration for expose "{e.service_expose_name}" '
f'not found in service "{e.service}"')
target_path = Dict.find(conf, 'http.path')
target_port = Dict.find(conf, 'http.port')
if target_path is None or target_port is None:
raise StoryscriptError(
message=f'http.path or http.port is null '
f'for expose {e.service}/{e.service_expose_name}')
await Containers.expose_service(self, e)
async def start_services(self):
tasks = []
reusable_services = set()
for story_name in self.stories.keys():
story = Stories(self, story_name, self.logger)
line = story.first_line()
while line is not None:
line = story.line(line)
method = line['method']
try:
if method != 'execute':
continue
chain = Services.resolve_chain(story, line)
assert isinstance(chain[0], Service)
assert isinstance(chain[1], Command)
if Containers.is_service_reusable(story.app, line):
# Simple cache to not unnecessarily make more calls to
# Kubernetes. It's okay if we don't have this check
# though, since the underlying API handles this.
service = chain[0].name
if service in reusable_services:
continue
reusable_services.add(service)
if not Services.is_internal(chain[0].name, chain[1].name):
tasks.append(Services.start_container(story, line))
finally:
line = line.get('next')
if len(tasks) > 0:
completed, pending = await asyncio.wait(tasks)
# Pending must never be greater than zero.
assert len(pending) == 0
for task in completed:
exc = task.exception()
if exc is not None:
raise exc
async def run_stories(self):
"""
Executes all the stories.
This enables the story to listen to pub/sub,
register with the gateway, and queue cron jobs.
"""
for story_name in self.entrypoint:
await Story.run(self, self.logger, story_name)
def add_subscription(self, sub_id: str,
streaming_service: StreamingService, event: str,
payload: dict):
self._subscriptions[sub_id] = Subscription(streaming_service, sub_id,
payload, event)
def get_subscription(self, sub_id: str):
return self._subscriptions.get(sub_id)
def remove_subscription(self, sub_id: str):
self._subscriptions.pop(sub_id)
async def clear_subscriptions_synapse(self):
url = f'http://{self.config.ASYNCY_SYNAPSE_HOST}:' \
f'{self.config.ASYNCY_SYNAPSE_PORT}/clear_all'
kwargs = {
'method': 'POST',
'body': json.dumps({'app_id': self.app_id}),
'headers': {
'Content-Type': 'application/json; charset=utf-8'
}
}
client = AsyncHTTPClient()
response = await HttpUtils.fetch_with_retry(3, self.logger, url,
client, kwargs)
if int(response.code / 100) == 2:
self.logger.debug(f'Unsubscribed all with Synapse!')
return True
else:
self.logger.error(f'Failed to unsubscribe with Synapse!')
return False
async def unsubscribe_all(self):
for sub_id, sub in self._subscriptions.items():
assert isinstance(sub, Subscription)
assert isinstance(sub.streaming_service, StreamingService)
conf = Dict.find(
self.services, f'{sub.streaming_service.name}'
f'.{ServiceConstants.config}'
f'.actions.{sub.streaming_service.command}'
f'.events.{sub.event}.http')
http_conf = conf.get('unsubscribe')
if not http_conf:
self.logger.debug(f'No unsubscribe call required for {sub}')
continue
url = f'http://{sub.streaming_service.hostname}' \
f':{http_conf.get("port", conf.get("port", 80))}' \
f'{http_conf["path"]}'
client = AsyncHTTPClient()
self.logger.debug(f'Unsubscribing {sub}...')
method = http_conf.get('method', 'post')
kwargs = {
'method': method.upper(),
'body': json.dumps(sub.payload['sub_body']),
'headers': {
'Content-Type': 'application/json; charset=utf-8'
}
}
response = await HttpUtils.fetch_with_retry(
3, self.logger, url, client, kwargs)
if int(response.code / 100) == 2:
self.logger.debug(f'Unsubscribed!')
else:
self.logger.error(f'Failed to unsubscribe {sub}!')
async def destroy(self):
"""
Unsubscribe from all existing subscriptions,
and delete the namespace.
"""
await self.clear_subscriptions_synapse()
await self.unsubscribe_all()
class Drheader:
"""
Core functionality for DrHeader. This is where the magic happens
"""
error_types = {
1: 'Header not included in response',
2: 'Header should not be returned',
3: 'Value does not match security policy',
4: 'Must-Contain directive missed',
5: 'Must-Avoid directive included',
6: 'Must-Contain-One directive missed',
7: 'Directive not included in response',
8: 'Directive should not be returned'
}
def __init__(self,
url=None,
method="GET",
headers=None,
status_code=None,
params=None,
request_headers=None,
verify=True):
"""
NOTE: at least one param required.
:param url: (optional) URL of target
:type url: str
:param method: (optional) Method to use when doing the request
:type method: str
:param headers: (optional) Override headers
:type headers: dict
:param status_code: Override status code
:type status_code: int
:param params: Request params
:type params: dict
:param request_headers: Request headers
:type request_headers: dict
:param verify: Verify the server's TLS certificate
:type verify: bool or str
"""
if request_headers is None:
request_headers = {}
if isinstance(headers, str):
headers = json.loads(headers)
elif url and not headers:
headers, status_code = self._get_headers(url, method, params,
request_headers, verify)
self.status_code = status_code
self.headers = CaseInsensitiveDict(headers)
self.anomalies = []
self.url = url
self.delimiter = ';'
self.report = []
@staticmethod
def _get_headers(url, method, params, request_headers, verify):
"""
Get headers for specified url.
:param url: URL of target
:type url: str
:param method: (optional) Method to use when doing the request
:type method: str
:param params: Request params
:type params: dict
:param request_headers: Request headers
:type request_headers: dict
:param verify: Verify the server's TLS certificate
:type verify: bool or str
:return: headers, status_code
:rtype: dict, int
"""
if validators.url(url):
req_obj = getattr(requests, method.lower())
r = req_obj(url,
data=params,
headers=request_headers,
verify=verify)
headers = r.headers
if len(r.raw.headers.getlist('Set-Cookie')) > 0:
headers['set-cookie'] = r.raw.headers.getlist('Set-Cookie')
return headers, r.status_code
def analyze(self, rules=None):
"""
Analyze the currently loaded headers against provided rules.
:param rules: Override rules to compare headers against
:type rules: dict
:return: Audit report
:rtype: list
"""
for header, value in self.headers.items():
if type(value) == str:
self.headers[header] = value.lower()
if type(value) == list:
value = [item.lower() for item in value]
self.headers[header] = value
if not rules:
rules = load_rules()
for rule, config in rules.items():
self.__validate_rules(config, header=rule)
if 'Directives' in config and rule in self.headers:
for directive, d_config in config['Directives'].items():
self.__validate_rules(d_config,
header=rule,
directive=directive)
return self.report
def __validate_rule_and_value(self, expected_value, header, directive):
"""
Verify headers content matches provided config.
:param expected_value: Expected value of header.
:param header: Name of header
:param directive: Name of directive (optional)
:return:
"""
expected_value_list = [str(item).lower() for item in expected_value]
if len(expected_value_list) == 1:
expected_value_list = [
item.strip(' ')
for item in expected_value_list[0].split(self.delimiter)
]
if directive:
rule = directive
headers = _to_dict(self.headers[header], ';', ' ')
else:
rule = header
headers = self.headers
if rule not in headers:
self.__add_report_item(severity='high',
error_type=7 if directive else 1,
header=header,
directive=directive,
expected=expected_value_list)
else:
rule_list = [
item.strip(' ') for item in headers[rule].split(self.delimiter)
]
if not all(elem in expected_value_list for elem in rule_list):
self.__add_report_item(severity='high',
error_type=3,
header=header,
directive=directive,
expected=expected_value_list,
value=headers[rule])
def __validate_not_exists(self, header, directive):
"""
Verify specified rule does not exist in loaded headers.
:param header: Name of header
:param directive: Name of directive (optional)
"""
if directive:
rule = directive
headers = _to_dict(self.headers[header], ';', ' ')
else:
rule = header
headers = self.headers
if rule in headers:
self.__add_report_item(severity='high',
error_type=8 if directive else 2,
header=header,
directive=directive)
def __validate_exists(self, header, directive):
"""
Verify specified rule exists in loaded headers.
:param header: Name of header
:param directive: Name of directive (optional)
"""
if directive:
rule = directive
headers = _to_dict(self.headers[header], ';', ' ')
else:
rule = header
headers = self.headers
if rule not in headers:
self.__add_report_item(severity='high',
error_type=7 if directive else 1,
header=header,
directive=directive)
return rule in headers # Return value to prevent subsequent avoid/contain checks if the header is not present
def __validate_must_avoid(self, config, header, directive):
"""
Verify specified values do not exist in loaded headers.
:param config: Configuration rule-set to use
:param header: Name of header
:param directive: Name of directive (optional)
"""
if directive:
rule = directive
header_value = _to_dict(self.headers[header], ';', ' ')[rule]
else:
rule = header
header_value = self.headers[rule]
config['Must-Avoid'] = [item.lower() for item in config['Must-Avoid']]
for avoid_value in config['Must-Avoid']:
if avoid_value in header_value and rule not in self.anomalies:
if rule.lower() == 'content-security-policy':
policy = _to_dict(self.headers[header], ';', ' ')
non_compliant_values = [
item for item in list(policy.values())
if avoid_value in item
]
indices = [
list(policy.values()).index(item)
for item in non_compliant_values
]
for index in indices:
self.__add_report_item(severity='medium',
error_type=5,
header=header,
directive=list(
policy.keys())[index],
avoid=config['Must-Avoid'],
value=avoid_value)
else:
self.__add_report_item(severity='medium',
error_type=5,
header=header,
directive=directive,
avoid=config['Must-Avoid'],
value=avoid_value)
def __validate_must_contain(self, config, header, directive):
"""
Verify the provided header contains certain params.
:param config: Configuration rule-set to use
:param header: Name of header
:param directive: Name of directive (optional)
"""
if directive:
rule = directive
header_value = _to_dict(self.headers[header], ';', ' ')[rule]
else:
rule = header
header_value = self.headers[rule]
if 'Must-Contain-One' in config:
config['Must-Contain-One'] = [
item.lower() for item in config['Must-Contain-One']
]
contain_values = header_value.split(
' ') if directive else header_value.split(self.delimiter)
does_contain = False
for contain_value in contain_values:
contain_value = contain_value.lstrip()
if contain_value in config['Must-Contain-One']:
does_contain = True
break
if not does_contain:
self.__add_report_item(severity='high',
error_type=6,
header=header,
directive=directive,
expected=config['Must-Contain-One'],
value=config['Must-Contain-One'])
elif 'Must-Contain' in config:
config['Must-Contain'] = [
item.lower() for item in config['Must-Contain']
]
if header.lower() == 'set-cookie':
for cookie in self.headers[header]:
for contain_value in config['Must-Contain']:
if contain_value not in cookie:
self.__add_report_item(
severity='high'
if contain_value == 'secure' else 'medium',
error_type=4,
header=header,
expected=config['Must-Contain'],
value=contain_value,
cookie=cookie)
else:
for contain_value in config['Must-Contain']:
if contain_value not in header_value and rule not in self.anomalies:
self.__add_report_item(severity='medium',
error_type=4,
header=header,
directive=directive,
expected=config['Must-Contain'],
value=contain_value)
def __validate_rules(self, config, header, directive=None):
"""
Entry point for validation.
:param config: Configuration rule-set to use
:param header: Name of header
:param directive: Name of directive (optional)
"""
try:
self.delimiter = config['Delimiter']
except KeyError:
self.delimiter = ';'
if config['Required'] is True or (config['Required'] == 'Optional'
and header in self.headers):
if config['Enforce']:
self.__validate_rule_and_value(config['Value'], header,
directive)
else:
exists = self.__validate_exists(header, directive)
if exists:
if 'Must-Contain-One' in config or 'Must-Contain' in config:
self.__validate_must_contain(config, header, directive)
if 'Must-Avoid' in config:
self.__validate_must_avoid(config, header, directive)
elif config['Required'] is False:
self.__validate_not_exists(header, directive)
def __add_report_item(self,
severity,
error_type,
header,
directive=None,
expected=None,
avoid=None,
value='',
cookie=''):
"""
Add a entry to report.
:param severity: [low, medium, high]
:type severity: str
:param error_type: [1...6] related to error_types
:type error_type: int
:param expected: Expected value of header
:param avoid: Avoid value of header
:param value: Current value of header
:param cookie: Value of cookie (if applicable)
"""
if directive:
error = {
'rule': header + ' - ' + directive,
'severity': severity,
'message': self.error_types[error_type]
}
else:
error = {
'rule': header,
'severity': severity,
'message': self.error_types[error_type]
}
if expected:
error['expected'] = expected
error['delimiter'] = self.delimiter
if avoid:
error['avoid'] = avoid
error['delimiter'] = self.delimiter
if error_type == 3:
error['value'] = value
elif error_type in (4, 5, 6):
if header.lower() == 'set-cookie':
error['value'] = cookie
else:
if directive:
error['value'] = _to_dict(self.headers[header], ';',
' ')[directive].strip('\'')
else:
error['value'] = self.headers[header]
error['anomaly'] = value
self.report.append(error)
class Request:
def __init__(self, method, url, data=None, headers=None):
"""Request objects
:param method: an HTTP method
:param url: a full URL including query string if needed
:param data: data to be sent in the body of POST request
:param headers: dict representing headers. Case-insensitive
Content-Type header should be given at least if data exists
"""
self.method = method.upper()
self.url = url
if self.method == 'GET':
self.data = None
elif self.method == 'POST':
self.data = data
else:
raise NotImplementedError('{}: unknown method'.format(self.method))
r = urlparse(self.url)
if r.scheme not in ('http', ):
raise NotImplementedError('{}: not supported'.format(r.scheme))
self.hostname = r.hostname
self.port = r.port if r.port else 80
# set request path
self.path = r.path or '/'
# append params and query to path if exist
if r.params:
self.path += ';' + r.params
if r.query:
self.path += '?' + r.query
# make headers
self.headers = CaseInsensitiveDict()
self.headers['Host'] = self.hostname
self.headers['Agent'] = 'httpcli'
self.headers['Accept'] = '*/*'
self.headers['Connection'] = 'keep-alive'
if headers:
for key, value in headers.items():
self.headers[key] = value
if self.data: # for PUT method
if not self.headers.get('content-type', None):
raise ValueError('No Content-type header specified')
self.headers['content-length'] = str(len(self.data))
def open(self):
"""Open an HTTP session. Then, send/receive request/response.
"""
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((self.hostname, self.port))
self.file = file = self.sock.makefile('rwb')
headers_list = [
key + ': ' + value for key, value in self.headers.items()
]
headers_str = '\r\n'.join(headers_list) + '\r\n'
template = '{method} {path} HTTP/1.1\r\n{headers}\r\n'
message = template.format(method=self.method,
path=self.path,
headers=headers_str)
logging.debug('Request message:\n{}'.format(message))
file.write(message.encode('utf-8'))
if self.data:
file.write(self.data.encode('utf-8'))
file.flush()
response = Response(self)
# read status line
response.status_code = response.read_status()
logging.debug('status code: {}'.format(response.status_code))
# read headers
response.read_headers()
logging.debug('Response headers:\n{}'.format(response.headers))
# read contents
response.read_content()
if response.status_code >= 400: # HTTP error response
logging.warning(response.content)
return response
def close(self):
"""Close this HTTP session
"""
if not self.sock._closed:
self.file.close()
self.sock.close()
