def assert_request_equal(self, expected, real_request): method, path = expected[:2] if urlparse(path).scheme: match_path = real_request["full_path"] else: match_path = real_request["path"] self.assertEqual((method, path), (real_request["method"], match_path)) if len(expected) > 2: body = expected[2] real_request["expected"] = body err_msg = "Body mismatch for %(method)s %(path)s, " "expected %(expected)r, and got %(body)r" % real_request self.orig_assertEqual(body, real_request["body"], err_msg) if len(expected) > 3: headers = CaseInsensitiveDict(expected[3]) for key, value in headers.items(): real_request["key"] = key real_request["expected_value"] = value real_request["value"] = real_request["headers"].get(key) err_msg = ( "Header mismatch on %(key)r, " "expected %(expected_value)r and got %(value)r " "for %(method)s %(path)s %(headers)r" % real_request ) self.orig_assertEqual(value, real_request["value"], err_msg) real_request["extra_headers"] = dict( (key, value) for key, value in real_request["headers"].items() if key not in headers ) if real_request["extra_headers"]: self.fail( "Received unexpected headers for %(method)s " "%(path)s, got %(extra_headers)r" % real_request )
class Response: """The object that each response must be packed into before sending. Same reason as the Request object. """ __slots__ = ['version', 'status', 'headers', 'body'] def __init__(self): self.version = HTTP1_1 self.status = 200 self.headers = CID() self.body = bytes() def __repr__(self): return '<Response %s %s>' % (self.status, self.reason) @property def reason(self): """Third argument in response status line""" return responses[self.status] def decode(self): """decode response""" decode_msg(self) def encode(self): """encode response""" encode_msg(self) def prepare(self) -> bytes: """Prepare the response for socket transmission""" self.headers['content-length'] = str(len(self.body)) return '{version} {status} {reason}{headers}\r\n\r\n'.format( version=self.version, status=self.status, reason=self.reason, headers=''.join( '\r\n%s: %s' % (k, v) for k, v in self.headers.items()) ).encode() + self.body
def assert_request_equal(self, expected, real_request): method, path = expected[:2] if urlparse(path).scheme: match_path = real_request['full_path'] else: match_path = real_request['path'] self.assertEqual((method, path), (real_request['method'], match_path)) if len(expected) > 2: body = expected[2] real_request['expected'] = body err_msg = 'Body mismatch for %(method)s %(path)s, ' \ 'expected %(expected)r, and got %(body)r' % real_request self.orig_assertEqual(body, real_request['body'], err_msg) if len(expected) > 3: headers = CaseInsensitiveDict(expected[3]) for key, value in headers.items(): real_request['key'] = key real_request['expected_value'] = value real_request['value'] = real_request['headers'].get(key) err_msg = ('Header mismatch on %(key)r, ' 'expected %(expected_value)r and got %(value)r ' 'for %(method)s %(path)s %(headers)r' % real_request) self.orig_assertEqual(value, real_request['value'], err_msg) real_request['extra_headers'] = dict( (key, value) for key, value in real_request['headers'].items() if key not in headers) if real_request['extra_headers']: self.fail('Received unexpected headers for %(method)s ' '%(path)s, got %(extra_headers)r' % real_request)
def assert_request_equal(self, expected, real_request): method, path = expected[:2] if urlparse(path).scheme: match_path = real_request['full_path'] else: match_path = real_request['path'] self.assertEqual((method, path), (real_request['method'], match_path)) if len(expected) > 2: body = expected[2] real_request['expected'] = body err_msg = 'Body mismatch for %(method)s %(path)s, ' \ 'expected %(expected)r, and got %(body)r' % real_request self.orig_assertEqual(body, real_request['body'], err_msg) if len(expected) > 3: headers = CaseInsensitiveDict(expected[3]) for key, value in headers.items(): real_request['key'] = key real_request['expected_value'] = value real_request['value'] = real_request['headers'].get(key) err_msg = ( 'Header mismatch on %(key)r, ' 'expected %(expected_value)r and got %(value)r ' 'for %(method)s %(path)s %(headers)r' % real_request) self.orig_assertEqual(value, real_request['value'], err_msg) real_request['extra_headers'] = dict( (key, value) for key, value in real_request['headers'].items() if key not in headers) if real_request['extra_headers']: self.fail('Received unexpected headers for %(method)s ' '%(path)s, got %(extra_headers)r' % real_request)
class Request: """The object that the server must pack all requests into. This is necessary to support multiple search apis.""" __slots__ = ['method', 'url', 'version', 'headers', 'body'] def __init__(self): self.version = HTTP1_1 self.headers = CID() self.url = None # type: URL self.body = bytes() self.method = str() def __repr__(self): return '<Request %s %s>' % (self.url, self.method) def decode(self): """decode request""" decode_msg(self) def encode(self): """encode request""" encode_msg(self) def prepare(self) -> bytes: """Prepare the request for socket transmission""" self.headers['content-length'] = str(len(self.body)) headers = ''.join('\r\n%s: %s' % (k, v) for k, v in self.headers.items()) return '{method} {url} {version}{headers}\r\n\r\n'.format( method=self.method, url=str(self.url), version=self.version, headers=headers).encode() + self.body
def test_preserve_last_key_case(self): cid = CaseInsensitiveDict({"Accept": "application/json", "user-Agent": "requests"}) cid.update({"ACCEPT": "application/json"}) cid["USER-AGENT"] = "requests" keyset = frozenset(["ACCEPT", "USER-AGENT"]) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset
def set_extra_headers(self, headers): header_dict = CaseInsensitiveDict(headers) if 'Reply-To' in header_dict: self.data["ReplyTo"] = header_dict.pop('Reply-To') self.data["Headers"] = [ {"Name": key, "Value": value} for key, value in header_dict.items() ]
def set_extra_headers(self, headers): header_dict = CaseInsensitiveDict(headers) if 'Reply-To' in header_dict: self.data["ReplyTo"] = header_dict.pop('Reply-To') self.data["Headers"] = [ {"Name": key, "Value": value} for key, value in header_dict.items() ]
def test_preserve_key_case(self): cid = CaseInsensitiveDict({ 'Accept': 'application/json', 'user-Agent': 'requests', }) keyset = frozenset(['Accept', 'user-Agent']) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset
def test_preserve_key_case(self): cid = CaseInsensitiveDict({ 'Accept': 'application/json', 'user-Agent': 'requests', }) keyset = frozenset(['Accept', 'user-Agent']) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset
def to_list(self, json_string: str, filters: CaseInsensitiveDict = None) -> list: """TODO""" ret_list = [] for value in json.loads(json_string).values(): if not filters or all(k in value and value[k] == v for k, v in filters.items()): ret_list.append(self.row_to_entity(value)) return ret_list
class SSDPResponse(object): """A container class for received SSDP responses.""" def __init__(self, response): """Initializes the SSDPResponse instance based on a response string""" super(BasicSSDPServiceDiscoverer.SSDPResponse, self).__init__() # Initialize fields self.Location = None self.USN = None self.ST = None self.Headers = CaseInsensitiveDict() # Parse response self._fromString(response) def _fromString(self, str): """Parses a response string and assigns values to the SSDPResponse object. :param str str: The string to parse.""" # Lazy method to parse all http-headers h = CaseInsensitiveDict({k.lower(): v for k, v in dict(re.findall(r'(?P<name>.*?): (?P<value>.*?)\r\n', str)).items()}) self.Headers = h # Set major fields if 'location' in h: self.Location = h['location'] if 'USN' in h: self.USN = h['USN'] if 'ST' in h: self.ST = h['ST'] def __repr__(self): return '<SSDPResponse from %s at %s; Headers: %s>' % (self.USN, self.Location, self.Headers.__repr__()) def __hash__(self): if self.USN is not None: return hash(self.USN) return hash(tuple(self.Headers.items())) def __eq__(self, other): if self is not None and other is None: return False if not isinstance(other, self.__class__): return False return hash(self) == hash(other) def __ne__(self, other): return not self.__eq__(other)
def clean_response_headers(self, headers: structures.CaseInsensitiveDict) -> dict: new_headers = dict(headers) ignored_headers = self.ignored_response_headers for key, value in headers.items(): if key.lower() in ignored_headers: continue new_headers[key] = value return new_headers
def test_preserve_last_key_case(self): cid = CaseInsensitiveDict({ 'Accept': 'application/json', 'user-Agent': 'requests', }) cid.update({'ACCEPT': 'application/json'}) cid['USER-AGENT'] = 'requests' keyset = frozenset(['ACCEPT', 'USER-AGENT']) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset
def test_preserve_last_key_case(self): cid = CaseInsensitiveDict({ 'Accept': 'application/json', 'user-Agent': 'requests', }) cid.update({'ACCEPT': 'application/json'}) cid['USER-AGENT'] = 'requests' keyset = frozenset(['ACCEPT', 'USER-AGENT']) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset
def verify_signature(self, query_parameters): """Verify the signature provided with the query parameters. http://docs.shopify.com/api/tutorials/oauth example usage:: from shopify_trois import Credentials from shopify_trois.engines import Json as Shopify from urllib.parse import parse_qsl credentials = Credentials( api_key='your_api_key', scope=['read_orders'], secret='your_app_secret' ) shopify = Shopify(shop_name="your_store_name", credentials=\ credentials) query_parameters = parse_qsl("code=238420989938cb70a609f6ece2e2586\ b&shop=yourstore.myshopify.com×tamp=1373382939&\ signature=6fb122e33c21851c465345b8cb97245e") if not shopify.verify_signature(query_parameters): raise Exception("invalid signature") credentials.code = dict(query_parameters).get('code') shopify.setup_access_token() :returns: Returns True if the signature is valid. """ params = CaseInsensitiveDict(query_parameters) signature = params.pop("signature", None) calculated = ["%s=%s" % (k, v) for k, v in params.items()] calculated.sort() calculated = "".join(calculated) calculated = "{secret}{calculated}".format( secret=self.credentials.secret, calculated=calculated ) md5 = hashlib.md5() md5.update(calculated.encode('utf-8')) produced = md5.hexdigest() return produced == signature
def verify_signature(self, query_parameters): """Verify the signature provided with the query parameters. http://docs.shopify.com/api/tutorials/oauth example usage:: from shopify_trois import Credentials from shopify_trois.engines import Json as Shopify from urllib.parse import parse_qsl credentials = Credentials( api_key='your_api_key', scope=['read_orders'], secret='your_app_secret' ) shopify = Shopify(shop_name="your_store_name", credentials=\ credentials) query_parameters = parse_qsl("code=238420989938cb70a609f6ece2e2586\ b&shop=yourstore.myshopify.com×tamp=1373382939&\ signature=6fb122e33c21851c465345b8cb97245e") if not shopify.verify_signature(query_parameters): raise Exception("invalid signature") credentials.code = dict(query_parameters).get('code') shopify.setup_access_token() :returns: Returns True if the signature is valid. """ params = CaseInsensitiveDict(query_parameters) signature = params.pop("signature", None) calculated = ["%s=%s" % (k, v) for k, v in params.items()] calculated.sort() calculated = "".join(calculated) calculated = "{secret}{calculated}".format( secret=self.credentials.secret, calculated=calculated) md5 = hashlib.md5() md5.update(calculated.encode('utf-8')) produced = md5.hexdigest() return produced == signature
def _prepare_request(self, command, json, opcode_name, fetch_list, **kwargs): params = CaseInsensitiveDict(**kwargs) params.update({ 'apiKey': self.key, opcode_name: command, }) if json: params['response'] = 'json' if 'page' in kwargs or fetch_list: params.setdefault('pagesize', PAGE_SIZE) kind = 'params' if self.method == 'get' else 'data' return kind, {k: v for k, v in params.items()}
def dict_subset(input_list, dic_data: CaseInsensitiveDict) -> CaseInsensitiveDict: for name in input_list: if name not in [key.strip().lower() for key in dic_data.keys()]: logging.warning( f"provided entry with the name - {name} is not " f"valid The program will disregard this input") input_list.remove(name) sub_data = { key: value for key, value in dic_data.items() if key.strip().lower() in [item.strip().lower() for item in input_list] } return CaseInsensitiveDict(data=sub_data)
def test_inject_into_non_sampled_context(self): carrier = CaseInsensitiveDict() AwsXRayPropagatorTest.XRAY_PROPAGATOR.inject( carrier, build_test_current_context(), ) injected_items = set(carrier.items()) expected_items = set( CaseInsensitiveDict({ TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0" }).items()) self.assertEqual(injected_items, expected_items)
def process_headers( self, headers: CaseInsensitiveDict = None, content_type: ContentType = ContentType.APPLICATION_JSON, content_length: int = 0) -> None: """TODO""" headers = MockServerHandler.remove_reserved_headers(headers) if headers and len(headers) > 0: for key, value in headers.items(): self.send_header(key, value) self.send_header("Server", 'MockServer v{}'.format(self.parent.version)) self.send_header("Date", self.date_time_string()) self.send_header("Content-Type", str(content_type)) self.send_header("Content-Length", str(content_length)) self.end_headers()
def test_inject_into_context_with_non_default_state(self): carrier = CaseInsensitiveDict() AwsXRayPropagatorTest.XRAY_PROPAGATOR.inject( carrier, build_test_current_context(trace_state=TraceState([("foo", "bar")])), ) # TODO: (NathanielRN) Assert trace state when the propagator supports it injected_items = set(carrier.items()) expected_items = set( CaseInsensitiveDict({ TRACE_HEADER_KEY: "Root=1-8a3c60f7-d188f8fa79d48a391a778fa6;Parent=53995c3f42cd8ad8;Sampled=0" }).items()) self.assertEqual(injected_items, expected_items)
def _user_preferences(self, _users_data: CaseInsensitiveDict) -> Dict: food_user_mappings = list() user_drinks_mappings = dict() for _user, pref in _users_data.items(): for item, val in pref.items(): if item == 'wont_eat': food_user_mappings.append( {key.strip().lower(): _user.strip() for key in val}) elif item == 'drinks': user_drinks_mappings[_user.strip()] = set( v.strip().lower() for v in val) return { 'bad_food_user_mappings': self._merge_dicts(food_user_mappings), 'user_drinks_mappings': user_drinks_mappings }
def _prepare_request(self, command, json=True, opcode_name='command', fetch_list=False, **kwargs): params = CaseInsensitiveDict(**kwargs) params.update({ 'apiKey': self.key, opcode_name: command, }) if json: params['response'] = 'json' if 'page' in kwargs or fetch_list: params.setdefault('pagesize', PAGE_SIZE) if 'expires' not in params and self.expiration.total_seconds() >= 0: params['signatureVersion'] = '3' tz = pytz.utc expires = tz.localize(datetime.utcnow() + self.expiration) params['expires'] = expires.astimezone(tz).strftime(EXPIRES_FORMAT) kind = 'params' if self.method == 'get' else 'data' return kind, dict(params.items())
class UserStore: """ Simple in-memory store for user info. Attribute names are case-insensitive. Users can be retrieved by id or username """ def __init__(self): self.users = CaseInsensitiveDict() # key = id self.names = CaseInsensitiveDict() # key = username def add(self, user_info): self._add_default_attributes(user_info) self.users[user_info.get('id')] = dict(user_info) self.names[user_info.get('username')] = dict(user_info) return id def get_by_id(self, id): return self.users.get(id, None) def get_by_name(self, username): return self.names.get(username, None) def update_scopes(self, username, scopes): self.names[username]['consented_scopes'] += ' ' + scopes def list(self): """ Returns a list of dictionaries representing users. password and consented_scopes attributes are not returned """ return [self._copy_user(u[1]) for u in self.users.items()] def _copy_user(self, user): d = copy.deepcopy(dict(user)) self._del_default_attributes(d) return d def _add_default_attributes(self, user_info): if 'consented_scopes' not in user_info: user_info['consented_scopes'] = '' def _del_default_attributes(self, dictionary): del dictionary['consented_scopes'] del dictionary['password']
def _get_email_headers_from_part(self, part): email_headers = list(part.items()) if not email_headers: return {} # Convert the header tuple into a dictionary headers = CaseInsensitiveDict() # assume duplicate header names with unique values. ex: received for x in email_headers: try: headers.setdefault(x[0].lower().replace('-', '_').replace(' ', '_'), []).append(x[1]) except Exception as e: error_msg = self._get_error_message_from_exception(e) err = "Error occurred while converting the header tuple into a dictionary" self.debug_print("{}. {}".format(err, error_msg)) headers = {k.lower(): '\n'.join(v) for k, v in headers.items()} return dict(headers)
class Response: """The object that each response must be packed into before sending. Same reason as the Request object. """ __slots__ = ['version', 'status', 'headers', 'body'] def __init__(self): self.version = HTTP1_1 self.status = 200 self.headers = CID() self.body = bytes() def __repr__(self): return '<Response %s %s>' % (self.status, self.reason) @property def reason(self): """Third argument in response status line""" return responses[self.status] def unpack(self): """Preprocess http message (e.g decode)""" if self.headers.get('content-encoding', '') == 'gzip': self.body = gzip.decompress(self.body) def prepare(self, request: Request) -> bytes: """Prepare the response for socket transmission""" if 'gzip' in request.headers.get('accept-encoding', ''): self.headers['content-encoding'] = 'gzip' self.body = gzip.compress(self.body) self.headers['content-length'] = str(len(self.body)) return '{version} {status} {reason}{headers}\r\n\r\n'.format( version=self.version, status=self.status, reason=self.reason, headers=''.join( '\r\n%s: %s' % (k, v) for k, v in self.headers.items())).encode() + self.body
def _prepare_request(self, command, json=True, opcode_name='command', fetch_list=False, **kwargs): params = CaseInsensitiveDict(**kwargs) params.update({ 'apiKey': self.key, opcode_name: command, }) if json: params['response'] = 'json' if 'page' in kwargs or fetch_list: params.setdefault('pagesize', PAGE_SIZE) if 'expires' not in params and self.expiration.total_seconds() >= 0: params['signatureVersion'] = '3' tz = pytz.utc expires = tz.localize(datetime.utcnow() + self.expiration) params['expires'] = expires.astimezone(tz).strftime(EXPIRES_FORMAT) kind = 'params' if self.method == 'get' else 'data' return kind, dict(params.items())
def sniffit(): """ Perform an HTTP/HTTPS request to the address that the user specifid :return: TODO Make the Google Verification a separate module with annotion """ parsed_url = urlparse(request.json["url"]) app.logger.info(request.remote_addr + " " + parsed_url.netloc) # Processing the headers to be sent to the URL that the user defined in the interface. # What we are doing here is making sure the the user can't override some headers that we want to force such as # X-Forwarded-For. request_headers = CaseInsensitiveDict({header["key"]: header["value"] for header in request.json["headers"]}) request_headers["X-Forwarded-For"] = request.remote_addr request_headers["X-Anti-Abuse"] = app.config.get("ABUSE_CONTACT") request_headers = {string.capwords(k, "-"): v for (k, v) in request_headers.items()} # Request Parameters if type(request.json["parameters"]) is list: request_parameters = "&".join([cgi.escape(header["key"])+"="+cgi.escape(header["value"]) for header in request.json["parameters"]]) else: request_parameters = request.json["parameters"] # Base Response JSON response_json = {'success': False, 'sniffed': None, 'messages': []} try: if string.lower(request.json["method"]) in ["get", "head", "options"]: response = requests.request(request.json["method"], request.json["url"], verify=False, params=request_parameters, headers=request_headers) else: response = requests.request(request.json["method"], request.json["url"], verify=False, data=request_parameters, headers=request_headers) # I prefer to have the capitalized headers in the frontend # This will convert the headers from 'content-type' to 'Content-Type' response_headers = {string.capwords(k, "-"): v for (k, v) in response.headers.items()} # This is for the adrministrators only so there is no need for the end-user to see this request_headers.pop("X-Anti-Abuse") request_headers.pop("X-Forwarded-For") # Create a history of redirects to inform the user redirections = [{"url": redirect.headers["location"]} for redirect in response.history] # Geo Location ipaddress = socket.gethostbyname(parsed_url.netloc) geolocation_response = requests.get("http://ip-api.com/json/" + ipaddress); response_json["success"] = True response_json["showRecaptcha"] = recaptcha_handler.is_token_invalid() response_json["sniffed"] = { 'headers': { 'response': response_headers, 'request': request_headers }, 'ipaddress': ipaddress, 'geolocation': geolocation_response.json(), 'ssl': None, 'redirect': redirections, 'body': base64.b64encode(cgi.escape(response.text.encode("UTF-8"))), 'size': response.headers.get("content-length", False), 'ssize': len(response.text.encode("UTF-8")), 'elapsed': response.elapsed.total_seconds(), 'status': { "reason": response.reason, "code": str(response.status_code) } } except Exception as e: raise RequestFailedException(repr(e)) return jsonify(response_json)
class Blink: """Class to initialize communication.""" def __init__( self, username=None, password=None, cred_file=None, refresh_rate=DEFAULT_REFRESH, motion_interval=DEFAULT_MOTION_INTERVAL, legacy_subdomain=False, no_prompt=False, persist_key=None, device_id="Blinkpy", ): """ Initialize Blink system. :param username: Blink username (usually email address) :param password: Blink password :param cred_file: JSON formatted file to store credentials. If username and password are given, file is ignored. Otherwise, username and password are loaded from file. :param refresh_rate: Refresh rate of blink information. Defaults to 15 (seconds) :param motion_interval: How far back to register motion in minutes. Defaults to last refresh time. Useful for preventing motion_detected property from de-asserting too quickly. :param legacy_subdomain: Set to TRUE to use old 'rest.region' endpoints (only use if you are having api issues). :param no_prompt: Set to TRUE if using an implementation that needs to suppress command-line output. :param persist_key: Location of persistant identifier. :param device_id: Identifier for the application. Default is 'Blinkpy'. This is used when logging in and should be changed to fit the implementation (ie. "Home Assistant" in a Home Assistant integration). """ self.login_handler = LoginHandler( username=username, password=password, cred_file=cred_file, persist_key=persist_key, device_id=device_id, ) self._token = None self._auth_header = None self._host = None self.account_id = None self.client_id = None self.network_ids = [] self.urls = None self.sync = CaseInsensitiveDict({}) self.region = None self.region_id = None self.last_refresh = None self.refresh_rate = refresh_rate self.session = create_session() self.networks = [] self.cameras = CaseInsensitiveDict({}) self.video_list = CaseInsensitiveDict({}) self.login_url = LOGIN_URLS[0] self.login_urls = [] self.motion_interval = motion_interval self.version = __version__ self.legacy = legacy_subdomain self.no_prompt = no_prompt self.available = False self.key_required = False self.login_response = {} @property def auth_header(self): """Return the authentication header.""" return self._auth_header def start(self): """ Perform full system setup. Method logs in and sets auth token, urls, and ids for future requests. Essentially this is just a wrapper function for ease of use. """ if not self.available: self.get_auth_token() if self.key_required and not self.no_prompt: email = self.login_handler.data["username"] key = input("Enter code sent to {}: ".format(email)) result = self.login_handler.send_auth_key(self, key) self.key_required = not result self.setup_post_verify() elif not self.key_required: self.setup_post_verify() def setup_post_verify(self): """Initialize blink system after verification.""" camera_list = self.get_cameras() networks = self.get_ids() for network_name, network_id in networks.items(): if network_id not in camera_list.keys(): camera_list[network_id] = {} _LOGGER.warning("No cameras found for %s", network_name) sync_module = BlinkSyncModule( self, network_name, network_id, camera_list[network_id] ) sync_module.start() self.sync[network_name] = sync_module self.cameras = self.merge_cameras() self.available = self.refresh() self.key_required = False def login(self): """Perform server login. DEPRECATED.""" _LOGGER.warning( "Method is deprecated and will be removed in a future version. Please use the LoginHandler.login() method instead." ) return self.login_handler.login(self) def get_auth_token(self, is_retry=False): """Retrieve the authentication token from Blink.""" self.login_response = self.login_handler.login(self) if not self.login_response: self.available = False return False self.setup_params(self.login_response) if self.login_handler.check_key_required(self): self.key_required = True return self._auth_header def setup_params(self, response): """Retrieve blink parameters from login response.""" self.login_url = self.login_handler.login_url ((self.region_id, self.region),) = response["region"].items() self._host = "{}.{}".format(self.region_id, BLINK_URL) self._token = response["authtoken"]["authtoken"] self._auth_header = {"Host": self._host, "TOKEN_AUTH": self._token} self.urls = BlinkURLHandler(self.region_id, legacy=self.legacy) self.networks = self.get_networks() self.client_id = response["client"]["id"] self.account_id = response["account"]["id"] def get_networks(self): """Get network information.""" response = api.request_networks(self) try: return response["summary"] except KeyError: return None def get_ids(self): """Set the network ID and Account ID.""" all_networks = [] network_dict = {} for network, status in self.networks.items(): if status["onboarded"]: all_networks.append("{}".format(network)) network_dict[status["name"]] = network self.network_ids = all_networks return network_dict def get_cameras(self): """Retrieve a camera list for each onboarded network.""" response = api.request_homescreen(self) try: all_cameras = {} for camera in response["cameras"]: camera_network = str(camera["network_id"]) camera_name = camera["name"] camera_id = camera["id"] camera_info = {"name": camera_name, "id": camera_id} if camera_network not in all_cameras: all_cameras[camera_network] = [] all_cameras[camera_network].append(camera_info) return all_cameras except KeyError: _LOGGER.error("Initialization failue. Could not retrieve cameras.") return {} @Throttle(seconds=MIN_THROTTLE_TIME) def refresh(self, force_cache=False): """ Perform a system refresh. :param force_cache: Force an update of the camera cache """ if self.check_if_ok_to_update() or force_cache: for sync_name, sync_module in self.sync.items(): _LOGGER.debug("Attempting refresh of sync %s", sync_name) sync_module.refresh(force_cache=force_cache) if not force_cache: # Prevents rapid clearing of motion detect property self.last_refresh = int(time.time()) return True return False def check_if_ok_to_update(self): """Check if it is ok to perform an http request.""" current_time = int(time.time()) last_refresh = self.last_refresh if last_refresh is None: last_refresh = 0 if current_time >= (last_refresh + self.refresh_rate): return True return False def merge_cameras(self): """Merge all sync camera dicts into one.""" combined = CaseInsensitiveDict({}) for sync in self.sync: combined = merge_dicts(combined, self.sync[sync].cameras) return combined def download_videos(self, path, since=None, camera="all", stop=10, debug=False): """ Download all videos from server since specified time. :param path: Path to write files. /path/<cameraname>_<recorddate>.mp4 :param since: Date and time to get videos from. Ex: "2018/07/28 12:33:00" to retrieve videos since July 28th 2018 at 12:33:00 :param camera: Camera name to retrieve. Defaults to "all". Use a list for multiple cameras. :param stop: Page to stop on (~25 items per page. Default page 10). :param debug: Set to TRUE to prevent downloading of items. Instead of downloading, entries will be printed to log. """ if since is None: since_epochs = self.last_refresh else: parsed_datetime = parse(since, fuzzy=True) since_epochs = parsed_datetime.timestamp() formatted_date = get_time(time_to_convert=since_epochs) _LOGGER.info("Retrieving videos since %s", formatted_date) if not isinstance(camera, list): camera = [camera] for page in range(1, stop): response = api.request_videos(self, time=since_epochs, page=page) _LOGGER.debug("Processing page %s", page) try: result = response["media"] if not result: raise IndexError except (KeyError, IndexError): _LOGGER.info("No videos found on page %s. Exiting.", page) break self._parse_downloaded_items(result, camera, path, debug) def _parse_downloaded_items(self, result, camera, path, debug): """Parse downloaded videos.""" for item in result: try: created_at = item["created_at"] camera_name = item["device_name"] is_deleted = item["deleted"] address = item["media"] except KeyError: _LOGGER.info("Missing clip information, skipping...") continue if camera_name not in camera and "all" not in camera: _LOGGER.debug("Skipping videos for %s.", camera_name) continue if is_deleted: _LOGGER.debug("%s: %s is marked as deleted.", camera_name, address) continue clip_address = "{}{}".format(self.urls.base_url, address) filename = "{}-{}".format(camera_name, created_at) filename = "{}.mp4".format(slugify(filename)) filename = os.path.join(path, filename) if not debug: if os.path.isfile(filename): _LOGGER.info("%s already exists, skipping...", filename) continue response = api.http_get(self, url=clip_address, stream=True, json=False) with open(filename, "wb") as vidfile: copyfileobj(response.raw, vidfile) _LOGGER.info("Downloaded video to %s", filename) else: print( ("Camera: {}, Timestamp: {}, " "Address: {}, Filename: {}").format( camera_name, created_at, address, filename ) )
def test_preserve_key_case(self): cid = CaseInsensitiveDict({"Accept": "application/json", "user-Agent": "requests"}) keyset = frozenset(["Accept", "user-Agent"]) assert frozenset(i[0] for i in cid.items()) == keyset assert frozenset(cid.keys()) == keyset assert frozenset(cid) == keyset
class ScrapeConfig: PUBLIC_DATACENTER_POOL = 'public_datacenter_pool' PUBLIC_RESIDENTIAL_POOL = 'public_residential_pool' url: str retry: bool = True method: str = 'GET' country: Optional[str] = None render_js: bool = False cache: bool = False cache_clear: bool = False ssl: bool = False dns: bool = False asp: bool = False debug: bool = False raise_on_upstream_error: bool = True cache_ttl: Optional[int] = None proxy_pool: Optional[str] = None session: Optional[str] = None tags: Optional[List[str]] = None correlation_id: Optional[str] = None cookies: Optional[CaseInsensitiveDict] = None body: Optional[str] = None data: Optional[Dict] = None headers: Optional[CaseInsensitiveDict] = None graphql: Optional[str] = None js: str = None rendering_wait: int = None wait_for_selector: Optional[str] = None session_sticky_proxy: bool = True screenshots: Optional[Dict] = None webhook: Optional[str] = None def __init__(self, url: str, retry: bool = True, method: str = 'GET', country: Optional[str] = None, render_js: bool = False, cache: bool = False, cache_clear: bool = False, ssl: bool = False, dns: bool = False, asp: bool = False, debug: bool = False, raise_on_upstream_error: bool = True, cache_ttl: Optional[int] = None, proxy_pool: Optional[str] = None, session: Optional[str] = None, tags: Optional[Set[str]] = None, correlation_id: Optional[str] = None, cookies: Optional[CaseInsensitiveDict] = None, body: Optional[str] = None, data: Optional[Dict] = None, headers: Optional[Union[CaseInsensitiveDict, Dict[str, str]]] = None, graphql: Optional[str] = None, js: str = None, rendering_wait: int = None, wait_for_selector: Optional[str] = None, screenshots: Optional[Dict] = None, session_sticky_proxy: Optional[bool] = None, webhook: Optional[str] = None): assert (type(url) is str) if isinstance(tags, List): tags = set(tags) cookies = cookies or {} headers = headers or {} self.cookies = CaseInsensitiveDict(cookies) self.headers = CaseInsensitiveDict(headers) self.url = url self.retry = retry self.method = method self.country = country self.session_sticky_proxy = session_sticky_proxy self.render_js = render_js self.cache = cache self.cache_clear = cache_clear self.asp = asp self.webhook = webhook self.session = session self.debug = debug self.cache_ttl = cache_ttl self.proxy_pool = proxy_pool self.tags = tags or set() self.correlation_id = correlation_id self.wait_for_selector = wait_for_selector self.body = body self.data = data self.graphql = graphql self.js = js self.rendering_wait = rendering_wait self.raise_on_upstream_error = raise_on_upstream_error self.screenshots = screenshots self.key = None self.dns = dns self.ssl = ssl if cookies: _cookies = [] for name, value in cookies.items(): _cookies.append(name + '=' + value) if 'cookie' in self.headers: if self.headers['cookie'][-1] != ';': self.headers['cookie'] += ';' else: self.headers['cookie'] = '' self.headers['cookie'] += '; '.join(_cookies) if self.body and self.data: raise ScrapeConfigError( 'You cannot pass both parameters body and data. You must choose' ) if method in ['POST', 'PUT', 'PATCH']: if self.body is None and self.data is not None: if 'content-type' not in self.headers: self.headers[ 'content-type'] = 'application/x-www-form-urlencoded' self.body = urlencode(data) else: if self.headers['content-type'].find( 'application/json') != -1: self.body = json.dumps(data) elif self.headers['content-type'].find( 'application/x-www-form-urlencoded') != -1: self.body = urlencode(data) else: raise ScrapeConfigError( 'Content-Type "%s" not supported, use body parameter to pass pre encoded body according to your content type' % self.headers['content-type']) elif self.body is None and self.data is None: self.headers['content-type'] = 'text/plain' def _bool_to_http(self, _bool: bool) -> str: return 'true' if _bool is True else 'false' def generate_distributed_correlation_id(self): self.correlation_id = abs( hash('-'.join( [gethostname(), str(getpid()), str(currentThread().ident)]))) def to_api_params(self, key: str) -> Dict: params = { 'key': self.key if self.key is not None else key, 'url': quote(self.url) } if self.country is not None: params['country'] = self.country for name, value in self.headers.items(): params['headers[%s]' % name] = value if self.webhook is not None: params['webhook_name'] = self.webhook if self.render_js is True: params['render_js'] = self._bool_to_http(self.render_js) if self.wait_for_selector is not None: params['wait_for_selector'] = self.wait_for_selector if self.js: params['js'] = b64encode( self.js.encode('utf-8')).decode('utf-8') if self.rendering_wait: params['rendering_wait'] = self.rendering_wait if self.screenshots is not None: for name, element in self.screenshots.items(): params['screenshots[%s]' % name] = element else: if self.wait_for_selector is not None: logging.warning( 'Params "wait_for_selector" is ignored. Works only if render_js is enabled' ) if self.screenshots: logging.warning( 'Params "screenshots" is ignored. Works only if render_js is enabled' ) if self.js: logging.warning( 'Params "js" is ignored. Works only if render_js is enabled' ) if self.rendering_wait: logging.warning( 'Params "rendering_wait" is ignored. Works only if render_js is enabled' ) if self.asp is True: params['asp'] = self._bool_to_http(self.asp) if self.retry is False: params['retry'] = self._bool_to_http(self.retry) if self.cache is True: params['cache'] = self._bool_to_http(self.cache) if self.cache_clear is True: params['cache_clear'] = self._bool_to_http(self.cache_clear) if self.cache_ttl is not None: params['cache_ttl'] = self.cache_ttl else: if self.cache_clear is True: logging.warning( 'Params "cache_clear" is ignored. Works only if cache is enabled' ) if self.cache_ttl is not None: logging.warning( 'Params "cache_ttl" is ignored. Works only if cache is enabled' ) if self.dns is True: params['dns'] = self._bool_to_http(self.dns) if self.ssl is True: params['ssl'] = self._bool_to_http(self.ssl) if self.tags: params['tags'] = ','.join(self.tags) if self.correlation_id: params['correlation_id'] = self.correlation_id if self.session: params['session'] = self.session if self.session_sticky_proxy is True: # false by default params['session_sticky_proxy'] = self._bool_to_http( self.session_sticky_proxy) else: if self.session_sticky_proxy: logging.warning( 'Params "session_sticky_proxy" is ignored. Works only if session is enabled' ) if self.debug is True: params['debug'] = self._bool_to_http(self.debug) if self.graphql: params['graphql_query'] = quote(self.graphql) if self.proxy_pool is not None: params['proxy_pool'] = self.proxy_pool return params @staticmethod def from_exported_config(config: str) -> 'ScrapeConfig': try: from msgpack import loads as msgpack_loads except ImportError as e: print( 'You must install msgpack package - run: pip install "scrapfly-sdk[seepdup] or pip install msgpack' ) raise data = msgpack_loads(base64.b64decode(config)) headers = {} for name, value in data['headers'].items(): if isinstance(value, Iterable): headers[name] = '; '.join(value) else: headers[name] = value return ScrapeConfig(url=data['url'], retry=data['retry'], headers=headers, session=data['session'], session_sticky_proxy=data['session_sticky_proxy'], cache=data['cache'], cache_ttl=data['cache_ttl'], cache_clear=data['cache_clear'], render_js=data['render_js'], method=data['method'], asp=data['asp'], body=data['body'], ssl=data['ssl'], dns=data['dns'], country=data['country'], debug=data['debug'], correlation_id=data['correlation_id'], tags=data['tags'], graphql=data['graphql_query'], js=data['js'], rendering_wait=data['rendering_wait'], screenshots=data['screenshots'] or {}, proxy_pool=data['proxy_pool'])
class Blink(): """Class to initialize communication.""" def __init__(self, username=None, password=None, refresh_rate=REFRESH_RATE): """ Initialize Blink system. :param username: Blink username (usually email address) :param password: Blink password :param refresh_rate: Refresh rate of blink information. Defaults to 15 (seconds) """ self._username = username self._password = password self._token = None self._auth_header = None self._host = None self.account_id = None self.network_ids = [] self.urls = None self.sync = CaseInsensitiveDict({}) self.region = None self.region_id = None self.last_refresh = None self.refresh_rate = refresh_rate self.session = None self.networks = [] self.cameras = CaseInsensitiveDict({}) self._login_url = LOGIN_URL @property def auth_header(self): """Return the authentication header.""" return self._auth_header def start(self): """ Perform full system setup. Method logs in and sets auth token, urls, and ids for future requests. Essentially this is just a wrapper function for ease of use. """ if self._username is None or self._password is None: self.login() else: self.get_auth_token() networks = self.get_ids() for network_name, network_id in networks.items(): sync_module = BlinkSyncModule(self, network_name, network_id) sync_module.start() self.sync[network_name] = sync_module self.cameras = self.merge_cameras() def login(self): """Prompt user for username and password.""" self._username = input("Username:"******"Password:"******"Login successful!") return True _LOGGER.warning("Unable to login with %s.", self._username) return False def get_auth_token(self): """Retrieve the authentication token from Blink.""" if not isinstance(self._username, str): raise BlinkAuthenticationException(ERROR.USERNAME) if not isinstance(self._password, str): raise BlinkAuthenticationException(ERROR.PASSWORD) login_url = LOGIN_URL self.session = create_session() response = api.request_login(self, login_url, self._username, self._password) if response.status_code == 200: response = response.json() (self.region_id, self.region), = response['region'].items() else: _LOGGER.debug( ("Received response code %s " "when authenticating, " "trying new url"), response.status_code ) login_url = LOGIN_BACKUP_URL response = api.request_login(self, login_url, self._username, self._password) self.region_id = 'piri' self.region = "UNKNOWN" self._host = "{}.{}".format(self.region_id, BLINK_URL) self._token = response['authtoken']['authtoken'] self._auth_header = {'Host': self._host, 'TOKEN_AUTH': self._token} self.networks = response['networks'] self.urls = BlinkURLHandler(self.region_id) self._login_url = login_url return self._auth_header def get_ids(self): """Set the network ID and Account ID.""" response = api.request_networks(self) # Look for only onboarded network, flag warning if multiple # since it's unexpected all_networks = [] network_dict = {} for network, status in self.networks.items(): if status['onboarded']: all_networks.append('{}'.format(network)) network_dict[status['name']] = network # For the first onboarded network we find, grab the account id for resp in response['networks']: if str(resp['id']) in all_networks: self.account_id = resp['account_id'] break self.network_ids = all_networks return network_dict def refresh(self, force_cache=False): """ Perform a system refresh. :param force_cache: Force an update of the camera cache """ if self.check_if_ok_to_update() or force_cache: for sync_name, sync_module in self.sync.items(): _LOGGER.debug("Attempting refresh of sync %s", sync_name) sync_module.refresh(force_cache=force_cache) def check_if_ok_to_update(self): """Check if it is ok to perform an http request.""" current_time = int(time.time()) last_refresh = self.last_refresh if last_refresh is None: last_refresh = 0 if current_time >= (last_refresh + self.refresh_rate): self.last_refresh = current_time return True return False def merge_cameras(self): """Merge all sync camera dicts into one.""" combined = CaseInsensitiveDict({}) for sync in self.sync: combined = merge_dicts(combined, self.sync[sync].cameras) return combined
class tizgmusicproxy(object): """A class for logging into a Google Play Music account and retrieving song URLs. """ all_songs_album_title = "All Songs" thumbs_up_playlist_name = "Thumbs Up" def __init__(self, email, password, device_id): self.__gmusic = Mobileclient() self.__email = email self.__device_id = device_id self.logged_in = False self.queue = list() self.queue_index = -1 self.play_queue_order = list() self.play_modes = TizEnumeration(["NORMAL", "SHUFFLE"]) self.current_play_mode = self.play_modes.NORMAL self.now_playing_song = None userdir = os.path.expanduser('~') tizconfig = os.path.join(userdir, ".config/tizonia/." + email + ".auth_token") auth_token = "" if os.path.isfile(tizconfig): with open(tizconfig, "r") as f: auth_token = pickle.load(f) if auth_token: # 'Keep track of the auth token' workaround. See: # https://github.com/diraimondo/gmusicproxy/issues/34#issuecomment-147359198 print_msg("[Google Play Music] [Authenticating] : " \ "'with cached auth token'") self.__gmusic.android_id = device_id self.__gmusic.session._authtoken = auth_token self.__gmusic.session.is_authenticated = True try: self.__gmusic.get_registered_devices() except CallFailure: # The token has expired. Reset the client object print_wrn("[Google Play Music] [Authenticating] : " \ "'auth token expired'") self.__gmusic = Mobileclient() auth_token = "" if not auth_token: attempts = 0 print_nfo("[Google Play Music] [Authenticating] : " \ "'with user credentials'") while not self.logged_in and attempts < 3: self.logged_in = self.__gmusic.login(email, password, device_id) attempts += 1 with open(tizconfig, "a+") as f: f.truncate() pickle.dump(self.__gmusic.session._authtoken, f) self.library = CaseInsensitiveDict() self.song_map = CaseInsensitiveDict() self.playlists = CaseInsensitiveDict() self.stations = CaseInsensitiveDict() def logout(self): """ Reset the session to an unauthenticated, default state. """ self.__gmusic.logout() def set_play_mode(self, mode): """ Set the playback mode. :param mode: curren tvalid values are "NORMAL" and "SHUFFLE" """ self.current_play_mode = getattr(self.play_modes, mode) self.__update_play_queue_order() def current_song_title_and_artist(self): """ Retrieve the current track's title and artist name. """ logging.info("current_song_title_and_artist") song = self.now_playing_song if song: title = to_ascii(self.now_playing_song.get('title')) artist = to_ascii(self.now_playing_song.get('artist')) logging.info("Now playing %s by %s", title, artist) return artist, title else: return '', '' def current_song_album_and_duration(self): """ Retrieve the current track's album and duration. """ logging.info("current_song_album_and_duration") song = self.now_playing_song if song: album = to_ascii(self.now_playing_song.get('album')) duration = to_ascii \ (self.now_playing_song.get('durationMillis')) logging.info("album %s duration %s", album, duration) return album, int(duration) else: return '', 0 def current_track_and_album_total(self): """Return the current track number and the total number of tracks in the album, if known. """ logging.info("current_track_and_album_total") song = self.now_playing_song track = 0 total = 0 if song: try: track = self.now_playing_song['trackNumber'] total = self.now_playing_song['totalTrackCount'] logging.info("track number %s total tracks %s", track, total) except KeyError: logging.info("trackNumber or totalTrackCount : not found") else: logging.info("current_song_track_number_" "and_total_tracks : not found") return track, total def current_song_year(self): """ Return the current track's year of publication. """ logging.info("current_song_year") song = self.now_playing_song year = 0 if song: try: year = song['year'] logging.info("track year %s", year) except KeyError: logging.info("year : not found") else: logging.info("current_song_year : not found") return year def clear_queue(self): """ Clears the playback queue. """ self.queue = list() self.queue_index = -1 def enqueue_artist(self, arg): """ Search the user's library for tracks from the given artist and adds them to the playback queue. :param arg: an artist """ try: self.__update_local_library() artist = None if arg not in self.library.keys(): for name, art in self.library.iteritems(): if arg.lower() in name.lower(): artist = art print_wrn("[Google Play Music] '{0}' not found. " \ "Playing '{1}' instead." \ .format(arg.encode('utf-8'), \ name.encode('utf-8'))) break if not artist: # Play some random artist from the library random.seed() artist = random.choice(self.library.keys()) artist = self.library[artist] print_wrn("[Google Play Music] '{0}' not found. "\ "Feeling lucky?." \ .format(arg.encode('utf-8'))) else: artist = self.library[arg] tracks_added = 0 for album in artist: tracks_added += self.__enqueue_tracks(artist[album]) print_wrn("[Google Play Music] Playing '{0}'." \ .format(to_ascii(artist))) self.__update_play_queue_order() except KeyError: raise KeyError("Artist not found : {0}".format(arg)) def enqueue_album(self, arg): """ Search the user's library for albums with a given name and adds them to the playback queue. """ try: self.__update_local_library() album = None artist = None tentative_album = None tentative_artist = None for library_artist in self.library: for artist_album in self.library[library_artist]: print_nfo("[Google Play Music] [Album] '{0}'." \ .format(to_ascii(artist_album))) if not album: if arg.lower() == artist_album.lower(): album = artist_album artist = library_artist break if not tentative_album: if arg.lower() in artist_album.lower(): tentative_album = artist_album tentative_artist = library_artist if album: break if not album and tentative_album: album = tentative_album artist = tentative_artist print_wrn("[Google Play Music] '{0}' not found. " \ "Playing '{1}' instead." \ .format(arg.encode('utf-8'), \ album.encode('utf-8'))) if not album: # Play some random album from the library random.seed() artist = random.choice(self.library.keys()) album = random.choice(self.library[artist].keys()) print_wrn("[Google Play Music] '{0}' not found. "\ "Feeling lucky?." \ .format(arg.encode('utf-8'))) if not album: raise KeyError("Album not found : {0}".format(arg)) self.__enqueue_tracks(self.library[artist][album]) print_wrn("[Google Play Music] Playing '{0}'." \ .format(to_ascii(album))) self.__update_play_queue_order() except KeyError: raise KeyError("Album not found : {0}".format(arg)) def enqueue_playlist(self, arg): """Search the user's library for playlists with a given name and adds the tracks of the first match to the playback queue. Requires Unlimited subscription. """ try: self.__update_local_library() self.__update_playlists() self.__update_playlists_unlimited() playlist = None playlist_name = None for name, plist in self.playlists.items(): print_nfo("[Google Play Music] [Playlist] '{0}'." \ .format(to_ascii(name))) if arg not in self.playlists.keys(): for name, plist in self.playlists.iteritems(): if arg.lower() in name.lower(): playlist = plist playlist_name = name print_wrn("[Google Play Music] '{0}' not found. " \ "Playing '{1}' instead." \ .format(arg.encode('utf-8'), \ to_ascii(name))) break if not playlist: # Play some random playlist from the library random.seed() playlist_name = random.choice(self.playlists.keys()) playlist = self.playlists[playlist_name] print_wrn("[Google Play Music] '{0}' not found. "\ "Feeling lucky?." \ .format(arg.encode('utf-8'))) else: playlist_name = arg playlist = self.playlists[arg] self.__enqueue_tracks(playlist) print_wrn("[Google Play Music] Playing '{0}'." \ .format(to_ascii(playlist_name))) self.__update_play_queue_order() except KeyError: raise KeyError("Playlist not found : {0}".format(arg)) def enqueue_station_unlimited(self, arg): """Search the user's library for a station with a given name and add its tracks to the playback queue. Requires Unlimited subscription. """ try: # First try to find a suitable station in the user's library self.__enqueue_user_station_unlimited(arg) if not len(self.queue): # If no suitable station is found in the user's library, then # search google play unlimited for a potential match. self.__enqueue_station_unlimited(arg) if not len(self.queue): raise KeyError except KeyError: raise KeyError("Station not found : {0}".format(arg)) def enqueue_genre_unlimited(self, arg): """Search Unlimited for a genre with a given name and add its tracks to the playback queue. Requires Unlimited subscription. """ print_msg("[Google Play Music] [Retrieving genres] : '{0}'. " \ .format(self.__email)) try: all_genres = list() root_genres = self.__gmusic.get_genres() second_tier_genres = list() for root_genre in root_genres: second_tier_genres += self.__gmusic.get_genres(root_genre['id']) all_genres += root_genres all_genres += second_tier_genres for genre in all_genres: print_nfo("[Google Play Music] [Genre] '{0}'." \ .format(to_ascii(genre['name']))) genre = dict() if arg not in all_genres: genre = next((g for g in all_genres \ if arg.lower() in to_ascii(g['name']).lower()), \ None) tracks_added = 0 while not tracks_added: if not genre and len(all_genres): # Play some random genre from the search results random.seed() genre = random.choice(all_genres) print_wrn("[Google Play Music] '{0}' not found. "\ "Feeling lucky?." \ .format(arg.encode('utf-8'))) genre_name = genre['name'] genre_id = genre['id'] station_id = self.__gmusic.create_station(genre_name, \ None, None, None, genre_id) num_tracks = 200 tracks = self.__gmusic.get_station_tracks(station_id, num_tracks) tracks_added = self.__enqueue_tracks(tracks) logging.info("Added %d tracks from %s to queue", tracks_added, genre_name) if not tracks_added: # This will produce another iteration in the loop genre = None print_wrn("[Google Play Music] Playing '{0}'." \ .format(to_ascii(genre['name']))) self.__update_play_queue_order() except KeyError: raise KeyError("Genre not found : {0}".format(arg)) except CallFailure: raise RuntimeError("Operation requires an Unlimited subscription.") def enqueue_situation_unlimited(self, arg): """Search Unlimited for a situation with a given name and add its tracks to the playback queue. Requires Unlimited subscription. """ print_msg("[Google Play Music] [Retrieving situations] : '{0}'. " \ .format(self.__email)) try: self.__enqueue_situation_unlimited(arg) if not len(self.queue): raise KeyError logging.info("Added %d tracks from %s to queue", \ len(self.queue), arg) except KeyError: raise KeyError("Situation not found : {0}".format(arg)) except CallFailure: raise RuntimeError("Operation requires an Unlimited subscription.") def enqueue_artist_unlimited(self, arg): """Search Unlimited for an artist and adds the artist's 200 top tracks to the playback queue. Requires Unlimited subscription. """ try: artist = self.__gmusic_search(arg, 'artist') include_albums = False max_top_tracks = 200 max_rel_artist = 0 artist_tracks = dict() if artist: artist_tracks = self.__gmusic.get_artist_info \ (artist['artist']['artistId'], include_albums, max_top_tracks, max_rel_artist)['topTracks'] if not artist_tracks: raise KeyError tracks_added = self.__enqueue_tracks(artist_tracks) logging.info("Added %d tracks from %s to queue", \ tracks_added, arg) self.__update_play_queue_order() except KeyError: raise KeyError("Artist not found : {0}".format(arg)) except CallFailure: raise RuntimeError("Operation requires an Unlimited subscription.") def enqueue_album_unlimited(self, arg): """Search Unlimited for an album and add its tracks to the playback queue. Requires Unlimited subscription. """ try: album = self.__gmusic_search(arg, 'album') album_tracks = dict() if album: album_tracks = self.__gmusic.get_album_info \ (album['album']['albumId'])['tracks'] if not album_tracks: raise KeyError print_wrn("[Google Play Music] Playing '{0}'." \ .format((album['album']['name']).encode('utf-8'))) tracks_added = self.__enqueue_tracks(album_tracks) logging.info("Added %d tracks from %s to queue", \ tracks_added, arg) self.__update_play_queue_order() except KeyError: raise KeyError("Album not found : {0}".format(arg)) except CallFailure: raise RuntimeError("Operation requires an Unlimited subscription.") def enqueue_tracks_unlimited(self, arg): """ Search Unlimited for a track name and adds all the matching tracks to the playback queue. Requires Unlimited subscription. """ print_msg("[Google Play Music] [Retrieving library] : '{0}'. " \ .format(self.__email)) try: max_results = 200 track_hits = self.__gmusic.search(arg, max_results)['song_hits'] if not len(track_hits): # Do another search with an empty string track_hits = self.__gmusic.search("", max_results)['song_hits'] print_wrn("[Google Play Music] '{0}' not found. "\ "Feeling lucky?." \ .format(arg.encode('utf-8'))) tracks = list() for hit in track_hits: tracks.append(hit['track']) tracks_added = self.__enqueue_tracks(tracks) logging.info("Added %d tracks from %s to queue", \ tracks_added, arg) self.__update_play_queue_order() except KeyError: raise KeyError("Playlist not found : {0}".format(arg)) except CallFailure: raise RuntimeError("Operation requires an Unlimited subscription.") def enqueue_promoted_tracks_unlimited(self): """ Retrieve the url of the next track in the playback queue. """ try: tracks = self.__gmusic.get_promoted_songs() count = 0 for track in tracks: store_track = self.__gmusic.get_track_info(track['storeId']) if u'id' not in store_track.keys(): store_track[u'id'] = store_track['nid'] self.queue.append(store_track) count += 1 if count == 0: print_wrn("[Google Play Music] Operation requires " \ "an Unlimited subscription.") logging.info("Added %d Unlimited promoted tracks to queue", \ count) self.__update_play_queue_order() except CallFailure: raise RuntimeError("Operation requires an Unlimited subscription.") def next_url(self): """ Retrieve the url of the next track in the playback queue. """ if len(self.queue): self.queue_index += 1 if (self.queue_index < len(self.queue)) \ and (self.queue_index >= 0): next_song = self.queue[self.play_queue_order[self.queue_index]] return self.__retrieve_track_url(next_song) else: self.queue_index = -1 return self.next_url() else: return '' def prev_url(self): """ Retrieve the url of the previous track in the playback queue. """ if len(self.queue): self.queue_index -= 1 if (self.queue_index < len(self.queue)) \ and (self.queue_index >= 0): prev_song = self.queue[self.play_queue_order[self.queue_index]] return self.__retrieve_track_url(prev_song) else: self.queue_index = len(self.queue) return self.prev_url() else: return '' def __update_play_queue_order(self): """ Update the queue playback order. A sequential order is applied if the current play mode is "NORMAL" or a random order if current play mode is "SHUFFLE" """ total_tracks = len(self.queue) if total_tracks: if not len(self.play_queue_order): # Create a sequential play order, if empty self.play_queue_order = range(total_tracks) if self.current_play_mode == self.play_modes.SHUFFLE: random.shuffle(self.play_queue_order) print_nfo("[Google Play Music] [Tracks in queue] '{0}'." \ .format(total_tracks)) def __retrieve_track_url(self, song): """ Retrieve a song url """ song_url = self.__gmusic.get_stream_url(song['id'], self.__device_id) try: self.now_playing_song = song return song_url except AttributeError: logging.info("Could not retrieve the song url!") raise def __update_local_library(self): """ Retrieve the songs and albums from the user's library """ print_msg("[Google Play Music] [Retrieving library] : '{0}'. " \ .format(self.__email)) songs = self.__gmusic.get_all_songs() self.playlists[self.thumbs_up_playlist_name] = list() # Retrieve the user's song library for song in songs: if "rating" in song and song['rating'] == "5": self.playlists[self.thumbs_up_playlist_name].append(song) song_id = song['id'] song_artist = song['artist'] song_album = song['album'] self.song_map[song_id] = song if song_artist == "": song_artist = "Unknown Artist" if song_album == "": song_album = "Unknown Album" if song_artist not in self.library: self.library[song_artist] = CaseInsensitiveDict() self.library[song_artist][self.all_songs_album_title] = list() if song_album not in self.library[song_artist]: self.library[song_artist][song_album] = list() self.library[song_artist][song_album].append(song) self.library[song_artist][self.all_songs_album_title].append(song) # Sort albums by track number for artist in self.library.keys(): logging.info("Artist : %s", to_ascii(artist)) for album in self.library[artist].keys(): logging.info(" Album : %s", to_ascii(album)) if album == self.all_songs_album_title: sorted_album = sorted(self.library[artist][album], key=lambda k: k['title']) else: sorted_album = sorted(self.library[artist][album], key=lambda k: k.get('trackNumber', 0)) self.library[artist][album] = sorted_album def __update_stations_unlimited(self): """ Retrieve stations (Unlimited) """ self.stations.clear() stations = self.__gmusic.get_all_stations() self.stations[u"I'm Feeling Lucky"] = 'IFL' for station in stations: station_name = station['name'] logging.info("station name : %s", to_ascii(station_name)) self.stations[station_name] = station['id'] def __enqueue_user_station_unlimited(self, arg): """ Enqueue a user station (Unlimited) """ print_msg("[Google Play Music] [Station search "\ "in user's library] : '{0}'. " \ .format(self.__email)) self.__update_stations_unlimited() station_name = arg station_id = None for name, st_id in self.stations.iteritems(): print_nfo("[Google Play Music] [Station] '{0}'." \ .format(to_ascii(name))) if arg not in self.stations.keys(): for name, st_id in self.stations.iteritems(): if arg.lower() in name.lower(): station_id = st_id station_name = name break else: station_id = self.stations[arg] num_tracks = 200 tracks = list() if station_id: try: tracks = self.__gmusic.get_station_tracks(station_id, \ num_tracks) except KeyError: raise RuntimeError("Operation requires an " "Unlimited subscription.") tracks_added = self.__enqueue_tracks(tracks) if tracks_added: if arg != station_name: print_wrn("[Google Play Music] '{0}' not found. " \ "Playing '{1}' instead." \ .format(arg.encode('utf-8'), name.encode('utf-8'))) logging.info("Added %d tracks from %s to queue", tracks_added, arg) self.__update_play_queue_order() else: print_wrn("[Google Play Music] '{0}' has no tracks. " \ .format(station_name)) if not len(self.queue): print_wrn("[Google Play Music] '{0}' " \ "not found in the user's library. " \ .format(arg.encode('utf-8'))) def __enqueue_station_unlimited(self, arg, max_results=200, quiet=False): """Search for a station and enqueue all of its tracks (Unlimited) """ if not quiet: print_msg("[Google Play Music] [Station search in "\ "Google Play Music] : '{0}'. " \ .format(arg.encode('utf-8'))) try: station_name = arg station_id = None station = self.__gmusic_search(arg, 'station', max_results, quiet) if station: station = station['station'] station_name = station['name'] seed = station['seed'] seed_type = seed['seedType'] track_id = seed['trackId'] if seed_type == u'2' else None artist_id = seed['artistId'] if seed_type == u'3' else None album_id = seed['albumId'] if seed_type == u'4' else None genre_id = seed['genreId'] if seed_type == u'5' else None playlist_token = seed['playlistShareToken'] if seed_type == u'8' else None curated_station_id = seed['curatedStationId'] if seed_type == u'9' else None num_tracks = max_results tracks = list() try: station_id \ = self.__gmusic.create_station(station_name, \ track_id, \ artist_id, \ album_id, \ genre_id, \ playlist_token, \ curated_station_id) tracks \ = self.__gmusic.get_station_tracks(station_id, \ num_tracks) except KeyError: raise RuntimeError("Operation requires an " "Unlimited subscription.") tracks_added = self.__enqueue_tracks(tracks) if tracks_added: if not quiet: print_wrn("[Google Play Music] [Station] : '{0}'." \ .format(station_name.encode('utf-8'))) logging.info("Added %d tracks from %s to queue", \ tracks_added, arg.encode('utf-8')) self.__update_play_queue_order() except KeyError: raise KeyError("Station not found : {0}".format(arg)) def __enqueue_situation_unlimited(self, arg): """Search for a situation and enqueue all of its tracks (Unlimited) """ print_msg("[Google Play Music] [Situation search in "\ "Google Play Music] : '{0}'. " \ .format(arg.encode('utf-8'))) try: situation_hits = self.__gmusic.search(arg)['situation_hits'] if not len(situation_hits): # Do another search with an empty string situation_hits = self.__gmusic.search("")['situation_hits'] print_wrn("[Google Play Music] '{0}' not found. "\ "Feeling lucky?." \ .format(arg.encode('utf-8'))) situation = next((hit for hit in situation_hits \ if 'best_result' in hit.keys()), None) num_tracks = 200 if not situation and len(situation_hits): max_results = num_tracks / len(situation_hits) for hit in situation_hits: situation = hit['situation'] print_nfo("[Google Play Music] [Situation] '{0} : {1}'." \ .format((hit['situation']['title']).encode('utf-8'), (hit['situation']['description']).encode('utf-8'))) self.__enqueue_station_unlimited(situation['title'], max_results, True) if not situation: raise KeyError except KeyError: raise KeyError("Situation not found : {0}".format(arg)) def __enqueue_tracks(self, tracks): """ Add tracks to the playback queue """ count = 0 for track in tracks: if u'id' not in track.keys(): track[u'id'] = track['nid'] self.queue.append(track) count += 1 return count def __update_playlists(self): """ Retrieve the user's playlists """ plists = self.__gmusic.get_all_user_playlist_contents() for plist in plists: plist_name = plist['name'] logging.info("playlist name : %s", to_ascii(plist_name)) tracks = plist['tracks'] tracks.sort(key=itemgetter('creationTimestamp')) self.playlists[plist_name] = list() for track in tracks: try: song = self.song_map[track['trackId']] self.playlists[plist_name].append(song) except IndexError: pass def __update_playlists_unlimited(self): """ Retrieve shared playlists (Unlimited) """ plists_subscribed_to = [p for p in self.__gmusic.get_all_playlists() \ if p.get('type') == 'SHARED'] for plist in plists_subscribed_to: share_tok = plist['shareToken'] playlist_items \ = self.__gmusic.get_shared_playlist_contents(share_tok) plist_name = plist['name'] logging.info("shared playlist name : %s", to_ascii(plist_name)) self.playlists[plist_name] = list() for item in playlist_items: try: song = item['track'] song['id'] = item['trackId'] self.playlists[plist_name].append(song) except IndexError: pass def __gmusic_search(self, query, query_type, max_results=200, quiet=False): """ Search Google Play (Unlimited) """ search_results = self.__gmusic.search(query, max_results)[query_type + '_hits'] result = next((hit for hit in search_results \ if 'best_result' in hit.keys()), None) if not result and len(search_results): secondary_hit = None for hit in search_results: if not quiet: print_nfo("[Google Play Music] [{0}] '{1}'." \ .format(query_type.capitalize(), (hit[query_type]['name']).encode('utf-8'))) if query.lower() == \ to_ascii(hit[query_type]['name']).lower(): result = hit break if query.lower() in \ to_ascii(hit[query_type]['name']).lower(): secondary_hit = hit if not result and secondary_hit: result = secondary_hit if not result and not len(search_results): # Do another search with an empty string search_results = self.__gmusic.search("")[query_type + '_hits'] if not result and len(search_results): # Play some random result from the search results random.seed() result = random.choice(search_results) if not quiet: print_wrn("[Google Play Music] '{0}' not found. "\ "Feeling lucky?." \ .format(query.encode('utf-8'))) return result
def complete(current, previous, step, previous_step_metas, table_concordance=None, anteprevious=None, debug=False): current = copy.deepcopy(current) previous = copy.deepcopy(previous) table_concordance = CaseInsensitiveDict(table_concordance or dict()) DEBUG = debug or '--debug' in sys.argv def log(text): if DEBUG: print(text, file=sys.stderr) def exit(): raise Exception('[complete_articles] Fatal error') find_num = re.compile(r'-?[a-z]*(\d+)(-[ta][\dIV]+|\D)?$') oldnum = 0 oldstep = {} oldjson = [] oldstatus = {} oldartids = [] oldarts = [] oldsects = [] olddepot = None try: for line in previous: if line["type"] != "texte": oldjson.append(line) else: oldnum = int(find_num.search(line['id']).group(1)) olddepot = line['depot'] if line["type"] == "article": keys = list(line['alineas'].keys()) keys.sort() oldstep[line["titre"]] = [line['alineas'][k] for k in keys] oldstatus[line["titre"]] = line['statut'] oldartids.append(line["titre"]) oldarts.append((line["titre"], line)) elif line["type"] == "section": oldsects.append(line) except Exception as e: print(type(e), e, file=sys.stderr) print("Incorrect previous text: %s" % previous) exit() if previous_step_metas and not previous_step_metas.get("skip"): try: assert(previous_step_metas["type"] == "texte") oldnum = int(find_num.search(previous_step_metas['id']).group(1)) except Exception as e: print(type(e), e, file=sys.stderr) print("Incorrect previous step: %s" % previous_step_metas) exit() gdoldstep = None if anteprevious: gdoldstep = {} for line in anteprevious: if line["type"] == "article": keys = list(line['alineas'].keys()) keys.sort() gdoldstep[line["titre"]] = [line['alineas'][k] for k in keys] ALL_ARTICLES = [] def write_json(data): nonlocal ALL_ARTICLES ALL_ARTICLES.append(data) null_reg = re.compile(r'^$') re_mat_simple = re.compile(r'[IVXDCLM\d]') re_mat_complex = re.compile(r'L[O.\s]*[IVXDCLM\d]') re_mat_complex2 = re.compile(r'\d+-\d+') re_clean_art = re.compile(r'^"?Art\.?\s*', re.I) make_sta_reg = lambda x: re.compile(r'^("?Art[\s\.]*)?%s\s*(([\.°\-]+\s*)+)' % re_clean_art.sub('', x)) make_end_reg = lambda x, rich: re.compile(r'^%s[IVXDCLM\d\-]+([\-\.\s]+\d*)*((%s|[A-Z])\s*)*(\(|et\s|%s)' % ('("?[LA][LArRtTO\.\s]+)?' if rich else "", bister, x)) re_sect_chg = re.compile(r'^((chap|t)itre|volume|livre|tome|(sous-)?section)\s+[1-9IVXDC]', re.I) def get_mark_from_last(text, s, l="", sep="", enable_copyall=False, copyall=False): log("- GET Extract from " + s + " to " + l) res = [] try: start = make_sta_reg(s) except Exception as e: print('ERROR', type(e), e, s, l, file=sys.stderr) exit() rich = re_mat_complex.match(s) or re_mat_complex2.match(s) or not re_mat_simple.match(s) if l: last = make_sta_reg(l) re_end = None record = False for n, i in enumerate(text): matc = start.match(i) # log(" TEST: " + i[:50]) if re_end and (re_end.match(i) or re_sect_chg.match(i)): if l: re_end = make_end_reg(sep, rich) l = "" else: log(" --> END FOUND") record = False break elif matc: sep = matc.group(2).strip() log(" --> START FOUND " + sep) record = True if l: re_end = last else: re_end = make_end_reg(sep, rich) elif copyall is True: record = True re_end = null_reg if n == 0: i = "%s%s %s" % (s, ". -" if re_mat_simple.match(s) else sep, i) if record: log(" copy alinea: " + i[:10]) res.append(i) # retry and get everything as I before II added if not found if not res: if not l and enable_copyall and not copyall: log(" nothing found, grabbing all article now...") return get_mark_from_last(text, s, l, sep=sep, copyall=True) print('ERROR: could not retrieve', s, file=sys.stderr) return False return res get_alineas_text = lambda a: clean_text_for_diff([a[k] for k in sorted(a.keys())]) re_clean_et = re.compile(r'(\s*[\&,]\s*|\s+et\s+)+', re.I) re_suppr = re.compile(r'\W*suppr(ess|im)', re.I) re_confo = re.compile(r'\W*(conforme|non[\s\-]*modifi)', re.I) re_confo_with_txt = re.compile(r'\s*\(\s*(conforme|non[\s\-]*modifié)\s*\)\s*([\W]*\w+)', re.I) re_clean_subsec_space = re.compile(r'^("?[IVX0-9]{1,4}(\s+[a-z]+)?(\s+[A-Z]{1,4})?)\s*([\.°\-]+)\s*([^\s\)])', re.I) order = 1 cursec = {'id': ''} done_titre = False texte = None for line_i, line in enumerate(current): if not line or not "type" in line: sys.stderr.write("JSON badly formatted, missing field type: %s\n" % line) exit() if oldnum and 'source_text' in line and oldnum != line['source_text']: continue if line["type"] == "echec": texte["echec"] = True texte["expose"] = line["texte"] write_json(texte) for a in oldjson: write_json(a) break elif line["type"] == "texte": texte = dict(line) # check number of sections is the same as the final text #if texte["definitif"]: # assert len(oldsects) == len([x for x in current if x['type'] == 'section']) else: if not done_titre: write_json(texte) done_titre = True if line["type"] != "article": if texte['definitif']: try: cursec = oldsects.pop(0) assert(cursec["type_section"] == line["type_section"]) except: print("ERROR: Problem while renumbering sections: ", line['titre'], " is not ", cursec, '\n', file=sys.stderr) # exit() if line["id"] != cursec["id"]: log("DEBUG: Matched section %s (%s) with old section %s (%s)" % (line["id"], line.get('titre'), cursec["id"], cursec.get('titre'))) line["newid"] = line["id"] line["id"] = cursec["id"] write_json(line) else: keys = list(line['alineas'].keys()) keys.sort() alineas = [line['alineas'][k] for k in keys] mult = line['titre'].split(' à ') is_mult = (len(mult) > 1) if is_mult: st = mult[0].strip() ed = mult[1].strip() if re_suppr.match(line['statut']) or (len(alineas) == 1 and re_suppr.match(alineas[0])): if (st not in oldartids and ed not in oldartids) or (st in oldstatus and re_suppr.match(oldstatus[st]) and ed in oldstatus and re_suppr.match(oldstatus[ed])): log("DEBUG: SKIP already deleted articles %s to %s" % (st, ed)) continue log("DEBUG: Marking as deleted articles %s à %s" % (st, ed)) mult_type = "sup" elif re_confo.match(line['statut']) or (len(alineas) == 1 and re_confo.match(alineas[0])): log("DEBUG: Recovering art conformes %s à %s" % (st, ed)) mult_type = "conf" else: print("ERROR: Found multiple article which I don't know what to do with", line['titre'], line, file=sys.stderr) exit() line['titre'] = st cur = "" if texte['definitif']: try: # recover the old article and mark those deleted as deleted while True: _, oldart = oldarts[0] # first, mark the old articles as deleted via the concordance table if oldart['titre'].lower() in table_concordance: new_art = table_concordance[oldart['titre']] if 'suppr' in new_art: c, a = oldarts.pop(0) oldartids.remove(c) if olddepot: log("DEBUG: Marking art %s as supprimé (thanks to concordance table)" % c) a["order"] = order order += 1 write_json(a) continue # as a backup, use the detected status to wait for a non-deleted article if re_suppr.match(oldart['statut']): c, a = oldarts.pop(0) oldartids.remove(c) if olddepot: log("DEBUG: Marking art %s as supprimé (recovered)" % c) a["order"] = order order += 1 write_json(a) else: break except Exception as e: print("ERROR: Problem while renumbering articles", line, "\n", oldart, "\n", type(e), e, file=sys.stderr) exit() # detect matching errors if oldart['titre'] in table_concordance: new_art = table_concordance[oldart['titre']] if new_art.lower() != line['titre'].lower(): print("ERROR: true concordance is different: when parsing article '%s', we matched it with '%s' which should be matched to '%s' (from concordance table) " % (line['titre'] , oldart['titre'], new_art)) match = None for oldart_title, newart_title in table_concordance.items(): if newart_title.lower() == line['titre'].lower(): match = oldart_title print(' -> it should have been matched with article %s' % oldart_title) break else: print(' -> it should have been deleted') # if article not matching but here in the concordance table, introduce it as a new one # since it can correspond to an article deleted by the Senate in Nouvelle lecture # or an article introduced in CMP hémicycle # /!\ this can only happen during a lecture définitive or a CMP hémicycle if (step.get('stage') == 'l. définitive' or ( step.get('step') == 'hemicycle' and step.get('stage') == 'CMP') ) and match: log("DEBUG: Marking art %s as nouveau" % line['titre']) if "section" in line and cursec['id'] != line["section"]: line["section"] = cursec["id"] a = line a["order"] = order a["status"] = "nouveau" if a['titre'] != match: a['newtitre'] = a['titre'] a['titre'] = match order += 1 write_json(a) continue else: exit() log("DEBUG: article '%s' matched with old article '%s'" % (line['titre'], oldart['titre'])) oldtxt = get_alineas_text(oldart["alineas"]) txt = get_alineas_text(line["alineas"]) similarity = compute_similarity(oldtxt, txt) if similarity < 0.75 and not olddepot and not step.get('stage') == 'constitutionnalité': print("WARNING BIG DIFFERENCE BETWEEN RENUMBERED ARTICLE", oldart["titre"], "<->", line["titre"], len(oldtxt), "Vs", len(txt), "chars, similarity; %.2f" % similarity, file=sys.stderr) if line['titre'] != oldart['titre']: line['newtitre'] = line['titre'] line['titre'] = oldart['titre'] if "section" in line and cursec['id'] != line["section"]: line["section"] = cursec["id"] if oldarts: while oldarts: cur, a = oldarts.pop(0) if line['titre'] in oldartids or article_is_lower(cur, line['titre']): oldartids.remove(cur) else: oldarts.insert(0, (cur, a)) break if cur == line['titre']: break if a["statut"].startswith("conforme"): log("DEBUG: Recovering art conforme %s" % cur) a["statut"] = "conforme" a["order"] = order order += 1 write_json(a) elif not re_suppr.match(a["statut"]): # if the last line of text was some dots, it means that we should keep # the articles as-is if they are not deleted last_block_was_dots_and_not_an_article = False for block in reversed(current[:line_i]): if block['type'] == 'dots': last_block_was_dots_and_not_an_article = True break if block['type'] == 'article': break if last_block_was_dots_and_not_an_article: # ex: https://www.senat.fr/leg/ppl09-304.html log("DEBUG: Recovering art as non-modifié via dots %s" % cur) a["statut"] = "non modifié" a["order"] = order order += 1 write_json(a) else: log("DEBUG: Marking art %s as supprimé because it disappeared" % cur) a["statut"] = "supprimé" a["alineas"] = dict() a["order"] = order order += 1 write_json(a) if is_mult: if ed not in oldartids or cur != line['titre']: if mult_type == "sup": print("WARNING: could not find first or last part of multiple article to be removed:", line['titre'], "to", ed, "(last found:", cur+")", file=sys.stderr) continue print("ERROR: dealing with multiple article", line['titre'], "to", ed, "Could not find first or last part in last step (last found:", cur+")", file=sys.stderr) exit() while True: if mult_type == "sup" and not re_suppr.match(a["statut"]): log("DEBUG: Marking art %s as supprimé (mult)" % cur) a["statut"] = "supprimé" a["alineas"] = dict() a["order"] = order order += 1 write_json(a) elif mult_type == "conf": log("DEBUG: Recovering art conforme %s (mult)" % cur) a["statut"] = "conforme" a["order"] = order order += 1 write_json(a) if cur == ed or not oldarts: break cur, a = oldarts.pop(0) continue if (re_suppr.match(line["statut"]) or (len(alineas) == 1 and re_suppr.match(alineas[0]))) and (line['titre'] not in oldstatus or re_suppr.match(oldstatus[line['titre']])): continue # Clean empty articles with only "Non modifié" and include text from previous step if alineas and re_confo.match(alineas[0]) and alineas[0].endswith(')'): if not line['titre'] in oldstep: sys.stderr.write("WARNING: found repeated article %s missing from previous step: %s (article is ignored)\n" % (line['titre'], line['alineas'])) # ignore empty non-modified continue else: log("DEBUG: get back Art %s" % line['titre']) alineas = oldstep[line['titre']] gd_text = [] enable_copyall = True for j, text in enumerate(alineas): if "(Non modifi" in text and not line['titre'] in oldstep: sys.stderr.write("WARNING: found repeated article missing %s from previous step: %s\n" % (line['titre'], text)) elif re_confo_with_txt.search(text): text = re_confo_with_txt.sub(r' \2', text) text = re_clean_subsec_space.sub(r'\1\4 \5', text) gd_text.append(text) elif "(Non modifi" in text: part = re.split("\s*([\.°\-]+\s*)+\s*\(Non", text) if not part: log("ERROR trying to get non-modifiés") exit() pieces = re_clean_et.sub(',', part[0]) log("EXTRACT non-modifiés for " + line['titre'] + ": " + pieces) piece = [] for todo in pieces.split(','): # Extract series of non-modified subsections of articles from previous version. if " à " in todo: start = re.split(" à ", todo)[0] end = re.split(" à ", todo)[1] mark = get_mark_from_last(oldstep[line['titre']], start, end, sep=part[1:], enable_copyall=enable_copyall) if mark is False and gdoldstep: mark = get_mark_from_last(gdoldstep[line['titre']], start, end, sep=part[1:], enable_copyall=enable_copyall) if mark is False: exit() enable_copyall = False piece.extend(mark) # Extract set of non-modified subsections of articles from previous version. elif todo: mark = get_mark_from_last(oldstep[line['titre']], todo, sep=part[1:], enable_copyall=enable_copyall) if mark is False and gdoldstep: mark = get_mark_from_last(gdoldstep[line['titre']], todo, sep=part[1:], enable_copyall=enable_copyall) if mark is False: exit() enable_copyall = False piece.extend(mark) gd_text.extend(piece) else: gd_text.append(text) line['alineas'] = dict() line['order'] = order order += 1 for i, t in enumerate(gd_text): line['alineas']["%03d" % (i+1)] = t write_json(line) if texte['definitif'] and oldsects and oldarts: print("ERROR: %s sections left:\n%s" % (len(oldsects), oldsects), file=sys.stderr) #exit() while oldarts: cur, a = oldarts.pop(0) oldartids.remove(cur) if texte['definitif'] and not re_suppr.match(a["statut"]): print("ERROR: %s articles left:\n%s %s" % (len(oldarts)+1, cur, oldartids), file=sys.stderr) exit() if not texte.get('echec', '') and a["statut"].startswith("conforme"): log("DEBUG: Recovering art conforme %s (leftovers)" % cur) a["statut"] = "conforme" a["order"] = order order += 1 write_json(a) # do not keep already deleted articles but mark as deleted missing ones elif not re_suppr.match(a["statut"]) or texte.get('echec', ''): # if the last line of text was some dots, it means that we should keep # the articles as-is if they are not deleted if line['type'] == 'dots': # ex: https://www.senat.fr/leg/ppl09-304.html log("DEBUG: Recovering art as non-modifié via dots %s (leftovers)" % cur) a["statut"] = "non modifié" a["order"] = order order += 1 write_json(a) else: log("DEBUG: Marking art %s as supprimé (leftovers)" % cur) a["statut"] = "supprimé" a["alineas"] = dict() a["order"] = order order += 1 write_json(a) return ALL_ARTICLES
def _recommendation(self, venues_data: CaseInsensitiveDict, user_pref: Dict) -> Dict: logging.info( f"Generating a recommendation report to advise which venues should the team members go." ) places_to_avoid = list() for bad_food, _users in user_pref['bad_food_user_mappings'].items(): for u in _users: for venue, available_food in venues_data.items(): # Assumption: if the venue offers anything else but what the user won't eat, then # it is assumed that the user is comfortable in visiting that place. However, # if the venue only offers the very food that the user won't eat then it is assumed that # that venue should be avoided... list_of_foods_offered_by_venue = list( set(x.strip().lower() for x in available_food['food'])) if ([bad_food] == list_of_foods_offered_by_venue) or ( len(list_of_foods_offered_by_venue) == 0): # [bad_food] == list_of_foods_offered_by_venue will return true only when the # venue offers one type of dish and that happens to be the one that the user won't eat. # len(list_of_foods_offered_by_venue) == 0 means that venue doesn't offer anything at all, # hence it should be avoided. places_to_avoid.append({ venue: f'There is nothing for {u.split()[0]} to eat.' }) for _user, drinks in user_pref['user_drinks_mappings'].items(): for _v_name, available_drinks in venues_data.items(): set_of_drinks_offered_by_venue = set( x.strip().lower() for x in available_drinks['drinks']) if len( set(drinks).intersection( set_of_drinks_offered_by_venue)) == 0: # len(set(drinks).intersection(set_of_drinks_offered_by_venue)) == 0 will return True if # the venue offers none of the drinks that are desired by the users; In either case # this suggests that the team cannot go to that venue. places_to_avoid.append({ _v_name: f'There is nothing for {_user.split()[0]} to drink.' }) avoid_dict = self._merge_dicts(places_to_avoid, default_type=list) return { 'places_to_visit': [ v for v in venues_data.keys() if v not in [key for key in avoid_dict.keys()] ], 'places_to_avoid': [{ 'name': key, 'reason': value } for key, value in avoid_dict.items()] }
class Blink: """Class to initialize communication.""" def __init__( self, refresh_rate=DEFAULT_REFRESH, motion_interval=DEFAULT_MOTION_INTERVAL, no_owls=False, ): """ Initialize Blink system. :param refresh_rate: Refresh rate of blink information. Defaults to 15 (seconds) :param motion_interval: How far back to register motion in minutes. Defaults to last refresh time. Useful for preventing motion_detected property from de-asserting too quickly. :param no_owls: Disable searching for owl entries (blink mini cameras only known entity). Prevents an uneccessary API call if you don't have these in your network. """ self.auth = Auth() self.account_id = None self.client_id = None self.network_ids = [] self.urls = None self.sync = CaseInsensitiveDict({}) self.last_refresh = None self.refresh_rate = refresh_rate self.networks = [] self.cameras = CaseInsensitiveDict({}) self.video_list = CaseInsensitiveDict({}) self.motion_interval = motion_interval self.version = __version__ self.available = False self.key_required = False self.homescreen = {} self.no_owls = no_owls @util.Throttle(seconds=MIN_THROTTLE_TIME) def refresh(self, force=False, force_cache=False): """ Perform a system refresh. :param force: Used to override throttle, resets refresh :param force_cache: Used to force update without overriding throttle """ if self.check_if_ok_to_update() or force or force_cache: if not self.available: self.setup_post_verify() self.get_homescreen() for sync_name, sync_module in self.sync.items(): _LOGGER.debug("Attempting refresh of sync %s", sync_name) sync_module.refresh(force_cache=(force or force_cache)) if not force_cache: # Prevents rapid clearing of motion detect property self.last_refresh = int(time.time()) return True return False def start(self): """Perform full system setup.""" try: self.auth.startup() self.setup_login_ids() self.setup_urls() self.get_homescreen() except (LoginError, TokenRefreshFailed, BlinkSetupError): _LOGGER.error("Cannot setup Blink platform.") self.available = False return False self.key_required = self.auth.check_key_required() if self.key_required: if self.auth.no_prompt: return True self.setup_prompt_2fa() return self.setup_post_verify() def setup_prompt_2fa(self): """Prompt for 2FA.""" email = self.auth.data["username"] pin = input(f"Enter code sent to {email}: ") result = self.auth.send_auth_key(self, pin) self.key_required = not result def setup_post_verify(self): """Initialize blink system after verification.""" try: self.setup_networks() networks = self.setup_network_ids() cameras = self.setup_camera_list() except BlinkSetupError: self.available = False return False for name, network_id in networks.items(): sync_cameras = cameras.get(network_id, {}) self.setup_sync_module(name, network_id, sync_cameras) self.cameras = self.merge_cameras() self.available = True self.key_required = False return True def setup_sync_module(self, name, network_id, cameras): """Initialize a sync module.""" self.sync[name] = BlinkSyncModule(self, name, network_id, cameras) self.sync[name].start() def get_homescreen(self): """Get homecreen information.""" if self.no_owls: _LOGGER.debug("Skipping owl extraction.") self.homescreen = {} return self.homescreen = api.request_homescreen(self) def setup_owls(self): """Check for mini cameras.""" network_list = [] camera_list = [] try: for owl in self.homescreen["owls"]: name = owl["name"] network_id = str(owl["network_id"]) if network_id in self.network_ids: camera_list.append( {network_id: {"name": name, "id": network_id, "type": "mini"}} ) continue if owl["onboarded"]: network_list.append(str(network_id)) self.sync[name] = BlinkOwl(self, name, network_id, owl) self.sync[name].start() except KeyError: # No sync-less devices found pass self.network_ids.extend(network_list) return camera_list def setup_camera_list(self): """Create camera list for onboarded networks.""" all_cameras = {} response = api.request_camera_usage(self) try: for network in response["networks"]: camera_network = str(network["network_id"]) if camera_network not in all_cameras: all_cameras[camera_network] = [] for camera in network["cameras"]: all_cameras[camera_network].append( {"name": camera["name"], "id": camera["id"]} ) mini_cameras = self.setup_owls() for camera in mini_cameras: for network, camera_info in camera.items(): all_cameras[network].append(camera_info) return all_cameras except (KeyError, TypeError): _LOGGER.error("Unable to retrieve cameras from response %s", response) raise BlinkSetupError def setup_login_ids(self): """Retrieve login id numbers from login response.""" self.client_id = self.auth.client_id self.account_id = self.auth.account_id def setup_urls(self): """Create urls for api.""" try: self.urls = util.BlinkURLHandler(self.auth.region_id) except TypeError: _LOGGER.error( "Unable to extract region is from response %s", self.auth.login_response ) raise BlinkSetupError def setup_networks(self): """Get network information.""" response = api.request_networks(self) try: self.networks = response["summary"] except (KeyError, TypeError): raise BlinkSetupError def setup_network_ids(self): """Create the network ids for onboarded networks.""" all_networks = [] network_dict = {} try: for network, status in self.networks.items(): if status["onboarded"]: all_networks.append(f"{network}") network_dict[status["name"]] = network except AttributeError: _LOGGER.error( "Unable to retrieve network information from %s", self.networks ) raise BlinkSetupError self.network_ids = all_networks return network_dict def check_if_ok_to_update(self): """Check if it is ok to perform an http request.""" current_time = int(time.time()) last_refresh = self.last_refresh if last_refresh is None: last_refresh = 0 if current_time >= (last_refresh + self.refresh_rate): return True return False def merge_cameras(self): """Merge all sync camera dicts into one.""" combined = CaseInsensitiveDict({}) for sync in self.sync: combined = util.merge_dicts(combined, self.sync[sync].cameras) return combined def save(self, file_name): """Save login data to file.""" util.json_save(self.auth.login_attributes, file_name) def download_videos( self, path, since=None, camera="all", stop=10, delay=1, debug=False ): """ Download all videos from server since specified time. :param path: Path to write files. /path/<cameraname>_<recorddate>.mp4 :param since: Date and time to get videos from. Ex: "2018/07/28 12:33:00" to retrieve videos since July 28th 2018 at 12:33:00 :param camera: Camera name to retrieve. Defaults to "all". Use a list for multiple cameras. :param stop: Page to stop on (~25 items per page. Default page 10). :param delay: Number of seconds to wait in between subsequent video downloads. :param debug: Set to TRUE to prevent downloading of items. Instead of downloading, entries will be printed to log. """ if since is None: since_epochs = self.last_refresh else: parsed_datetime = parse(since, fuzzy=True) since_epochs = parsed_datetime.timestamp() formatted_date = util.get_time(time_to_convert=since_epochs) _LOGGER.info("Retrieving videos since %s", formatted_date) if not isinstance(camera, list): camera = [camera] for page in range(1, stop): response = api.request_videos(self, time=since_epochs, page=page) _LOGGER.debug("Processing page %s", page) try: result = response["media"] if not result: raise KeyError except (KeyError, TypeError): _LOGGER.info("No videos found on page %s. Exiting.", page) break self._parse_downloaded_items(result, camera, path, delay, debug) def _parse_downloaded_items(self, result, camera, path, delay, debug): """Parse downloaded videos.""" for item in result: try: created_at = item["created_at"] camera_name = item["device_name"] is_deleted = item["deleted"] address = item["media"] except KeyError: _LOGGER.info("Missing clip information, skipping...") continue if camera_name not in camera and "all" not in camera: _LOGGER.debug("Skipping videos for %s.", camera_name) continue if is_deleted: _LOGGER.debug("%s: %s is marked as deleted.", camera_name, address) continue clip_address = f"{self.urls.base_url}{address}" filename = f"{camera_name}-{created_at}" filename = f"{slugify(filename)}.mp4" filename = os.path.join(path, filename) if not debug: if os.path.isfile(filename): _LOGGER.info("%s already exists, skipping...", filename) continue response = api.http_get( self, url=clip_address, stream=True, json=False, timeout=TIMEOUT_MEDIA, ) with open(filename, "wb") as vidfile: copyfileobj(response.raw, vidfile) _LOGGER.info("Downloaded video to %s", filename) else: print( ( f"Camera: {camera_name}, Timestamp: {created_at}, " "Address: {address}, Filename: {filename}" ) ) if delay > 0: time.sleep(delay)
def complete(current, previous, step, table_concordance, anteprevious=None): current = copy.deepcopy(current) previous = copy.deepcopy(previous) table_concordance = CaseInsensitiveDict(table_concordance) DEBUG = '--debug' in sys.argv def log(text): if DEBUG: print(text, file=sys.stderr) def exit(): raise Exception('[complete_articles] Fatal error') find_num = re.compile(r'-[a-z]*(\d+)\D?$') oldnum = 0 oldstep = {} oldjson = [] oldstatus = {} oldartids = [] oldarts = [] oldsects = [] try: for line in previous: if line["type"] != "texte": oldjson.append(line) else: oldnum = int(find_num.search(line['id']).group(1)) olddepot = line['depot'] if line["type"] == "article": keys = list(line['alineas'].keys()) keys.sort() oldstep[line["titre"]] = [line['alineas'][k] for k in keys] oldstatus[line["titre"]] = line['statut'] oldartids.append(line["titre"]) oldarts.append((line["titre"], line)) elif line["type"] == "section": oldsects.append(line) except Exception as e: print(type(e), e, file=sys.stderr) log("No previous step found at %s" % sys.argv[2]) exit() gdoldstep = None if anteprevious: gdoldstep = {} for line in anteprevious: if line["type"] == "article": keys = list(line['alineas'].keys()) keys.sort() gdoldstep[line["titre"]] = [line['alineas'][k] for k in keys] ALL_ARTICLES = [] def write_json(data): nonlocal ALL_ARTICLES ALL_ARTICLES.append(data) null_reg = re.compile(r'^$') re_mat_simple = re.compile(r'[IVXDCLM\d]') re_mat_complex = re.compile(r'L[O.\s]*[IVXDCLM\d]') re_mat_complex2 = re.compile(r'\d+-\d+') re_clean_art = re.compile(r'^"?Art\.?\s*', re.I) make_sta_reg = lambda x: re.compile( r'^("?Art[\s\.]*)?%s\s*(([\.°\-]+\s*)+)' % re_clean_art.sub('', x)) make_end_reg = lambda x, rich: re.compile( r'^%s[IVXDCLM\d\-]+([\-\.\s]+\d*)*((%s|[A-Z])\s*)*(\(|et\s|%s)' % ('("?[LA][LArRtTO\.\s]+)?' if rich else "", bister, x)) re_sect_chg = re.compile( r'^((chap|t)itre|volume|livre|tome|(sous-)?section)\s+[1-9IVXDC]', re.I) def get_mark_from_last(text, s, l="", sep="", enable_copyall=False, copyall=False): log("- GET Extract from " + s + " to " + l) res = [] try: start = make_sta_reg(s) except Exception as e: print('ERROR', type(e), e, s, l, file=sys.stderr) exit() rich = re_mat_complex.match(s) or re_mat_complex2.match( s) or not re_mat_simple.match(s) if l: last = make_sta_reg(l) re_end = None record = False for n, i in enumerate(text): matc = start.match(i) # log(" TEST: " + i[:50]) if re_end and (re_end.match(i) or re_sect_chg.match(i)): if l: re_end = make_end_reg(sep, rich) l = "" else: log(" --> END FOUND") record = False break elif matc: sep = matc.group(2).strip() log(" --> START FOUND " + sep) record = True if l: re_end = last else: re_end = make_end_reg(sep, rich) elif copyall is True: record = True re_end = null_reg if n == 0: i = "%s%s %s" % (s, ". -" if re_mat_simple.match(s) else sep, i) if record: log(" copy alinea") res.append(i) # retry and get everything as I before II added if not found if not res: if not l and enable_copyall and not copyall: log(" nothing found, grabbing all article now...") return get_mark_from_last(text, s, l, sep=sep, copyall=True) print('ERROR: could not retrieve', s, file=sys.stderr) return False return res get_alineas_text = lambda a: clean_text_for_diff( [a[k] for k in sorted(a.keys())]) re_clean_et = re.compile(r'(\s*[\&,]\s*|\s+et\s+)+', re.I) re_suppr = re.compile(r'\W*suppr(ess|im)', re.I) re_confo = re.compile(r'\W*(conforme|non[\s\-]*modifi)', re.I) re_confo_with_txt = re.compile( r'\s*\(\s*(conforme|non[\s\-]*modifié)\s*\)\s*([\W]*\w+)', re.I) re_clean_subsec_space = re.compile( r'^("?[IVX0-9]{1,4}(\s+[a-z]+)?(\s+[A-Z]{1,4})?)\s*([\.°\-]+)\s*([^\s\)])', re.I) order = 1 cursec = {'id': ''} done_titre = False texte = None for line_i, line in enumerate(current): if not line or not "type" in line: sys.stderr.write("JSON badly formatted, missing field type: %s\n" % line) exit() if oldnum and 'source_text' in line and oldnum != line['source_text']: continue if line["type"] == "echec": texte["echec"] = True texte["expose"] = line["texte"] write_json(texte) for a in oldjson: write_json(a) break elif line["type"] == "texte": texte = dict(line) # check number of sections is the same as the final text #if texte["definitif"]: # assert len(oldsects) == len([x for x in current if x['type'] == 'section']) else: if not done_titre: write_json(texte) done_titre = True if line["type"] != "article": if texte['definitif']: try: cursec = oldsects.pop(0) assert (cursec["type_section"] == line["type_section"]) except: print("ERROR: Problem while renumbering sections: ", line['titre'], " is not ", cursec, '\n', file=sys.stderr) # exit() if line["id"] != cursec["id"]: log("DEBUG: Matched section %s (%s) with old section %s (%s)" % (line["id"], line.get('titre'), cursec["id"], cursec.get('titre'))) line["newid"] = line["id"] line["id"] = cursec["id"] write_json(line) else: keys = list(line['alineas'].keys()) keys.sort() alineas = [line['alineas'][k] for k in keys] mult = line['titre'].split(' à ') is_mult = (len(mult) > 1) if is_mult: st = mult[0].strip() ed = mult[1].strip() if re_suppr.match(line['statut']) or ( len(alineas) == 1 and re_suppr.match(alineas[0])): if (st not in oldartids and ed not in oldartids) or ( st in oldstatus and re_suppr.match( oldstatus[st]) and ed in oldstatus and re_suppr.match(oldstatus[ed])): log("DEBUG: SKIP already deleted articles %s to %s" % (st, ed)) continue log("DEBUG: Marking as deleted articles %s à %s" % (st, ed)) mult_type = "sup" elif re_confo.match(line['statut']) or ( len(alineas) == 1 and re_confo.match(alineas[0])): log("DEBUG: Recovering art conformes %s à %s" % (st, ed)) mult_type = "conf" else: print( "ERROR: Found multiple article which I don't know what to do with", line['titre'], line, file=sys.stderr) exit() line['titre'] = st cur = "" if texte['definitif']: try: # recover the old article and mark those deleted as deleted while True: _, oldart = oldarts[0] # first, mark the old articles as deleted via the concordance table if oldart['titre'].lower() in table_concordance: new_art = table_concordance[oldart['titre']] if 'suppr' in new_art: c, a = oldarts.pop(0) oldartids.remove(c) if olddepot: log("DEBUG: Marking art %s as supprimé (thanks to concordance table)" % c) a["order"] = order order += 1 write_json(a) continue # as a backup, use the detected status to wait for a non-deleted article if re_suppr.match(oldart['statut']): c, a = oldarts.pop(0) oldartids.remove(c) if olddepot: log("DEBUG: Marking art %s as supprimé (recovered)" % c) a["order"] = order order += 1 write_json(a) else: break except: print("ERROR: Problem while renumbering articles", line, "\n", oldart, file=sys.stderr) exit() # detect matching errors if oldart['titre'] in table_concordance: new_art = table_concordance[oldart['titre']] if new_art.lower() != line['titre'].lower(): print( "ERROR: true concordance is different: when parsing article '%s', we matched it with '%s' which should be matched to '%s' (from concordance table) " % (line['titre'], oldart['titre'], new_art)) match = None for oldart_title, newart_title in table_concordance.items( ): if newart_title.lower() == line['titre'].lower( ): match = oldart_title print( ' -> it should have been matched with article %s' % oldart_title) break else: print(' -> it should have been deleted') # if article not matching but here in the concordance table, introduce it as a new one # since it can correspond to an article deleted by the Senate in Nouvelle lecture # or an article introduced in CMP hémicycle # /!\ this can only happen during a lecture définitive or a CMP hémicycle if (step.get('stage') == 'l. définitive' or (step.get('step') == 'hemicycle' and step.get('stage') == 'CMP')) and match: log("DEBUG: Marking art %s as nouveau" % line['titre']) if "section" in line and cursec['id'] != line[ "section"]: line["section"] = cursec["id"] a = line a["order"] = order a["status"] = "nouveau" if a['titre'] != match: a['newtitre'] = a['titre'] a['titre'] = match order += 1 write_json(a) continue else: exit() log("DEBUG: article '%s' matched with old article '%s'" % (line['titre'], oldart['titre'])) oldtxt = get_alineas_text(oldart["alineas"]) txt = get_alineas_text(line["alineas"]) similarity = compute_similarity(oldtxt, txt) if similarity < 0.75 and not olddepot and not step.get( 'stage') == 'constitutionnalité': print( "WARNING BIG DIFFERENCE BETWEEN RENUMBERED ARTICLE", oldart["titre"], "<->", line["titre"], len(oldtxt), "Vs", len(txt), "chars, similarity; %.2f" % similarity, file=sys.stderr) if line['titre'] != oldart['titre']: line['newtitre'] = line['titre'] line['titre'] = oldart['titre'] if "section" in line and cursec['id'] != line["section"]: line["section"] = cursec["id"] if oldarts: while oldarts: cur, a = oldarts.pop(0) if line['titre'] in oldartids or article_is_lower( cur, line['titre']): oldartids.remove(cur) else: oldarts.insert(0, (cur, a)) break if cur == line['titre']: break if a["statut"].startswith("conforme"): log("DEBUG: Recovering art conforme %s" % cur) a["statut"] = "conforme" a["order"] = order order += 1 write_json(a) elif not re_suppr.match(a["statut"]): # if the last line of text was some dots, it means that we should keep # the articles as-is if they are not deleted last_block_was_dots_and_not_an_article = False for block in reversed(current[:line_i]): if block['type'] == 'dots': last_block_was_dots_and_not_an_article = True break if block['type'] == 'article': break if last_block_was_dots_and_not_an_article: # ex: https://www.senat.fr/leg/ppl09-304.html log("DEBUG: Recovering art as non-modifié via dots %s" % cur) a["statut"] = "non modifié" a["order"] = order order += 1 write_json(a) else: log("DEBUG: Marking art %s as supprimé because it disappeared" % cur) a["statut"] = "supprimé" a["alineas"] = dict() a["order"] = order order += 1 write_json(a) if is_mult: if ed not in oldartids or cur != line['titre']: if mult_type == "sup": print( "WARNING: could not find first or last part of multiple article to be removed:", line['titre'], "to", ed, "(last found:", cur + ")", file=sys.stderr) continue print( "ERROR: dealing with multiple article", line['titre'], "to", ed, "Could not find first or last part in last step (last found:", cur + ")", file=sys.stderr) exit() while True: if mult_type == "sup" and not re_suppr.match( a["statut"]): log("DEBUG: Marking art %s as supprimé (mult)" % cur) a["statut"] = "supprimé" a["alineas"] = dict() a["order"] = order order += 1 write_json(a) elif mult_type == "conf": log("DEBUG: Recovering art conforme %s (mult)" % cur) a["statut"] = "conforme" a["order"] = order order += 1 write_json(a) if cur == ed or not oldarts: break cur, a = oldarts.pop(0) continue if (re_suppr.match(line["statut"]) or (len(alineas) == 1 and re_suppr.match(alineas[0]))) and ( line['titre'] not in oldstatus or re_suppr.match(oldstatus[line['titre']])): continue # Clean empty articles with only "Non modifié" and include text from previous step if alineas and re_confo.match( alineas[0]) and alineas[0].endswith(')'): if not line['titre'] in oldstep: sys.stderr.write( "WARNING: found repeated article %s missing from previous step: %s (article is ignored)\n" % (line['titre'], line['alineas'])) # ignore empty non-modified continue else: log("DEBUG: get back Art %s" % line['titre']) alineas = oldstep[line['titre']] gd_text = [] enable_copyall = True for j, text in enumerate(alineas): if "(Non modifi" in text and not line['titre'] in oldstep: sys.stderr.write( "WARNING: found repeated article missing %s from previous step: %s\n" % (line['titre'], text)) elif re_confo_with_txt.search(text): text = re_confo_with_txt.sub(r' \2', text) text = re_clean_subsec_space.sub(r'\1\4 \5', text) gd_text.append(text) elif "(Non modifi" in text: part = re.split("\s*([\.°\-]+\s*)+\s*\(Non", text) if not part: log("ERROR trying to get non-modifiés") exit() pieces = re_clean_et.sub(',', part[0]) log("EXTRACT non-modifiés for " + line['titre'] + ": " + pieces) piece = [] for todo in pieces.split(','): # Extract series of non-modified subsections of articles from previous version. if " à " in todo: start = re.split(" à ", todo)[0] end = re.split(" à ", todo)[1] mark = get_mark_from_last( oldstep[line['titre']], start, end, sep=part[1:], enable_copyall=enable_copyall) if mark is False and gdoldstep: mark = get_mark_from_last( gdoldstep[line['titre']], start, end, sep=part[1:], enable_copyall=enable_copyall) if mark is False: exit() enable_copyall = False piece.extend(mark) # Extract set of non-modified subsections of articles from previous version. elif todo: mark = get_mark_from_last( oldstep[line['titre']], todo, sep=part[1:], enable_copyall=enable_copyall) if mark is False and gdoldstep: mark = get_mark_from_last( gdoldstep[line['titre']], todo, sep=part[1:], enable_copyall=enable_copyall) if mark is False: exit() enable_copyall = False piece.extend(mark) gd_text.extend(piece) else: gd_text.append(text) line['alineas'] = dict() line['order'] = order order += 1 for i, t in enumerate(gd_text): line['alineas']["%03d" % (i + 1)] = t write_json(line) if texte['definitif'] and oldsects and oldarts: print("ERROR: %s sections left:\n%s" % (len(oldsects), oldsects), file=sys.stderr) #exit() while oldarts: cur, a = oldarts.pop(0) oldartids.remove(cur) if texte['definitif'] and not re_suppr.match(a["statut"]): print("ERROR: %s articles left:\n%s %s" % (len(oldarts) + 1, cur, oldartids), file=sys.stderr) exit() if not texte.get('echec', '') and a["statut"].startswith("conforme"): log("DEBUG: Recovering art conforme %s (leftovers)" % cur) a["statut"] = "conforme" a["order"] = order order += 1 write_json(a) # do not keep already deleted articles but mark as deleted missing ones elif not re_suppr.match(a["statut"]) or texte.get('echec', ''): # if the last line of text was some dots, it means that we should keep # the articles as-is if they are not deleted if line['type'] == 'dots': # ex: https://www.senat.fr/leg/ppl09-304.html log("DEBUG: Recovering art as non-modifié via dots %s (leftovers)" % cur) a["statut"] = "non modifié" a["order"] = order order += 1 write_json(a) else: log("DEBUG: Marking art %s as supprimé (leftovers)" % cur) a["statut"] = "supprimé" a["alineas"] = dict() a["order"] = order order += 1 write_json(a) return ALL_ARTICLES
class MockRestURI(object): """Representation of a mock URI.""" def __init__(self, uri_dict): self._uri_dict = uri_dict self.name = uri_dict.get('name', None) self.fullpath = uri_dict['request']['path'] self._regexp = False self.path = urlparse.urlparse(self.fullpath).path self.querystring = urlparse.parse_qs( urlparse.urlparse(self.fullpath).query, keep_blank_values=True ) self.method = uri_dict['request'].get('method', None) self.headers = CaseInsensitiveDict(uri_dict['request'].get('headers', {})) self.return_code = int(uri_dict['response'].get('code', '200')) self.request_content_type = self.headers.get('Content-Type', '') self.response_location = uri_dict['response'].get('location', None) self.response_content = uri_dict['response'].get('content', "") self.wait = float(uri_dict['response'].get('wait', 0.0)) self.request_data = uri_dict['request'].get('data', None) self.encoding = uri_dict['request'].get("encoding", None) self.response_headers = uri_dict['response'].get("headers", {}) def match(self, request): """Does this URI match the request?""" # Match HTTP verb - GET, POST, PUT, DELETE, etc. if self.method is not None: if request.command.lower() != self.method.lower(): return False # Match path if self.path != urlparse.urlparse(request.path).path: return False # Match querystring if request.querystring() != self.querystring: return False # Match headers if self.headers is not None: for header_var, header_value in self.headers.items(): request_headers = CaseInsensitiveDict(request.headers) if request_headers.get(header_var) is None: return False req_maintext, req_pdict = cgi.parse_header(request_headers.get(header_var)) mock_maintext, mock_pdict = cgi.parse_header(header_value) if "boundary" in req_pdict and "boundary" in mock_pdict: req_pdict['boundary'] = "xxx" mock_pdict['boundary'] = "xxx" if req_maintext != mock_maintext: return False if mock_pdict != {}: if req_pdict != mock_pdict: return False # Match processed request data if self.request_data is not None: # Check if exact match before parsing if request.body != self.request_data: if self.request_content_type.startswith("application/json"): if json.loads(request.body) != json.loads(self.request_data): return False elif self.request_content_type.startswith("application/x-www-form-urlencoded"): if parse_qsl_as_dict(request.body) != parse_qsl_as_dict(self.request_data): return False elif self.request_content_type.startswith('application/xml'): actual_request_data_root = etree.fromstring(request.body) mock_request_data_root = etree.fromstring(self.request_data) if not xml_elements_equal(actual_request_data_root, mock_request_data_root): return False elif self.request_content_type.startswith("multipart/form-data"): ctype, pdict = cgi.parse_header(request.headers.get('Content-Type')) req_multipart = cgi.parse_multipart( io.BytesIO(request.body.encode('utf8')), {x: y.encode('utf8') for x, y in pdict.items()} ) ctype, pdict = cgi.parse_header(self.headers.get('Content-Type')) mock_multipart = cgi.parse_multipart( io.BytesIO(self.request_data.encode('utf8')), {x: y.encode('utf8') for x, y in pdict.items()} ) return mock_multipart == req_multipart else: if request.body != self.request_data: return False return True def querystring_string(self): # TODO : Refactor this and docstring. query = '' for key in self.querystring.keys(): for item in self.querystring[key]: query += str(key) + '=' + item + "&" query = query.rstrip("&") return "?" + query if query else "" def example_path(self): return xeger.xeger(self.path) if self._regexp else self.path + self.querystring_string() def return_code_description(self): return status_codes.CODES.get(self.return_code)[0] def request_data_values(self): if self.request_data is not None: return self.request_data.get('values', {}).iteritems() else: return [] def request_data_type(self): if self.request_data is not None: return self.request_data.get('encoding') else: return None
class Blink(): """Class to initialize communication.""" def __init__(self, username=None, password=None, refresh_rate=DEFAULT_REFRESH, motion_interval=DEFAULT_MOTION_INTERVAL, legacy_subdomain=False): """ Initialize Blink system. :param username: Blink username (usually email address) :param password: Blink password :param refresh_rate: Refresh rate of blink information. Defaults to 15 (seconds) :param motion_interval: How far back to register motion in minutes. Defaults to last refresh time. Useful for preventing motion_detected property from de-asserting too quickly. :param legacy_subdomain: Set to TRUE to use old 'rest.region' endpoints (only use if you are having api issues). """ self._username = username self._password = password self._token = None self._auth_header = None self._host = None self.account_id = None self.network_ids = [] self.urls = None self.sync = CaseInsensitiveDict({}) self.region = None self.region_id = None self.last_refresh = None self.refresh_rate = refresh_rate self.session = create_session() self.networks = [] self.cameras = CaseInsensitiveDict({}) self.video_list = CaseInsensitiveDict({}) self._login_url = LOGIN_URL self.login_urls = [] self.motion_interval = motion_interval self.version = __version__ self.legacy = legacy_subdomain @property def auth_header(self): """Return the authentication header.""" return self._auth_header def start(self): """ Perform full system setup. Method logs in and sets auth token, urls, and ids for future requests. Essentially this is just a wrapper function for ease of use. """ if self._username is None or self._password is None: if not self.login(): return elif not self.get_auth_token(): return camera_list = self.get_cameras() networks = self.get_ids() for network_name, network_id in networks.items(): if network_id not in camera_list.keys(): camera_list[network_id] = {} _LOGGER.warning("No cameras found for %s", network_name) sync_module = BlinkSyncModule(self, network_name, network_id, camera_list[network_id]) sync_module.start() self.sync[network_name] = sync_module self.cameras = self.merge_cameras() def login(self): """Prompt user for username and password.""" self._username = input("Username:"******"Password:"******"Login successful!") return True _LOGGER.warning("Unable to login with %s.", self._username) return False def get_auth_token(self, is_retry=False): """Retrieve the authentication token from Blink.""" if not isinstance(self._username, str): raise BlinkAuthenticationException(ERROR.USERNAME) if not isinstance(self._password, str): raise BlinkAuthenticationException(ERROR.PASSWORD) self.login_urls = [LOGIN_URL, OLD_LOGIN_URL, LOGIN_BACKUP_URL] response = self.login_request(is_retry=is_retry) if not response: return False self._host = "{}.{}".format(self.region_id, BLINK_URL) self._token = response['authtoken']['authtoken'] self.networks = response['networks'] self._auth_header = {'Host': self._host, 'TOKEN_AUTH': self._token} self.urls = BlinkURLHandler(self.region_id, legacy=self.legacy) return self._auth_header def login_request(self, is_retry=False): """Make a login request.""" try: login_url = self.login_urls.pop(0) except IndexError: _LOGGER.error("Could not login to blink servers.") return False _LOGGER.info("Attempting login with %s", login_url) response = api.request_login(self, login_url, self._username, self._password, is_retry=is_retry) try: if response.status_code != 200: response = self.login_request(is_retry=True) response = response.json() (self.region_id, self.region), = response['region'].items() except AttributeError: _LOGGER.error("Login API endpoint failed with response %s", response) return False except KeyError: _LOGGER.warning("Could not extract region info.") self.region_id = 'piri' self.region = 'UNKNOWN' self._login_url = login_url return response def get_ids(self): """Set the network ID and Account ID.""" response = api.request_networks(self) all_networks = [] network_dict = {} for network, status in self.networks.items(): if status['onboarded']: all_networks.append('{}'.format(network)) network_dict[status['name']] = network # For the first onboarded network we find, grab the account id for resp in response['networks']: if str(resp['id']) in all_networks: self.account_id = resp['account_id'] break self.network_ids = all_networks return network_dict def get_cameras(self): """Retrieve a camera list for each onboarded network.""" response = api.request_homescreen(self) try: all_cameras = {} for camera in response['cameras']: camera_network = str(camera['network_id']) camera_name = camera['name'] camera_id = camera['id'] camera_info = {'name': camera_name, 'id': camera_id} if camera_network not in all_cameras: all_cameras[camera_network] = [] all_cameras[camera_network].append(camera_info) return all_cameras except KeyError: _LOGGER.error("Initialization failue. Could not retrieve cameras.") return {} @Throttle(seconds=MIN_THROTTLE_TIME) def refresh(self, force_cache=False): """ Perform a system refresh. :param force_cache: Force an update of the camera cache """ if self.check_if_ok_to_update() or force_cache: for sync_name, sync_module in self.sync.items(): _LOGGER.debug("Attempting refresh of sync %s", sync_name) sync_module.refresh(force_cache=force_cache) if not force_cache: # Prevents rapid clearing of motion detect property self.last_refresh = int(time.time()) return True return False def check_if_ok_to_update(self): """Check if it is ok to perform an http request.""" current_time = int(time.time()) last_refresh = self.last_refresh if last_refresh is None: last_refresh = 0 if current_time >= (last_refresh + self.refresh_rate): return True return False def merge_cameras(self): """Merge all sync camera dicts into one.""" combined = CaseInsensitiveDict({}) for sync in self.sync: combined = merge_dicts(combined, self.sync[sync].cameras) return combined def download_videos(self, path, since=None, camera='all', stop=10, debug=False): """ Download all videos from server since specified time. :param path: Path to write files. /path/<cameraname>_<recorddate>.mp4 :param since: Date and time to get videos from. Ex: "2018/07/28 12:33:00" to retrieve videos since July 28th 2018 at 12:33:00 :param camera: Camera name to retrieve. Defaults to "all". Use a list for multiple cameras. :param stop: Page to stop on (~25 items per page. Default page 10). :param debug: Set to TRUE to prevent downloading of items. Instead of downloading, entries will be printed to log. """ if since is None: since_epochs = self.last_refresh else: parsed_datetime = parse(since, fuzzy=True) since_epochs = parsed_datetime.timestamp() formatted_date = get_time(time_to_convert=since_epochs) _LOGGER.info("Retrieving videos since %s", formatted_date) if not isinstance(camera, list): camera = [camera] for page in range(1, stop): response = api.request_videos(self, time=since_epochs, page=page) _LOGGER.debug("Processing page %s", page) try: result = response['media'] if not result: raise IndexError except (KeyError, IndexError): _LOGGER.info("No videos found on page %s. Exiting.", page) break self._parse_downloaded_items(result, camera, path, debug) def _parse_downloaded_items(self, result, camera, path, debug): """Parse downloaded videos.""" for item in result: try: created_at = item['created_at'] camera_name = item['device_name'] is_deleted = item['deleted'] address = item['media'] except KeyError: _LOGGER.info("Missing clip information, skipping...") continue if camera_name not in camera and 'all' not in camera: _LOGGER.debug("Skipping videos for %s.", camera_name) continue if is_deleted: _LOGGER.debug("%s: %s is marked as deleted.", camera_name, address) continue clip_address = "{}{}".format(self.urls.base_url, address) filename = "{}-{}".format(camera_name, created_at) filename = "{}.mp4".format(slugify(filename)) filename = os.path.join(path, filename) if not debug: if os.path.isfile(filename): _LOGGER.info("%s already exists, skipping...", filename) continue response = api.http_get(self, url=clip_address, stream=True, json=False) with open(filename, 'wb') as vidfile: copyfileobj(response.raw, vidfile) _LOGGER.info("Downloaded video to %s", filename) else: print(("Camera: {}, Timestamp: {}, " "Address: {}, Filename: {}").format( camera_name, created_at, address, filename))
def _decode_headers(headers: CaseInsensitiveDict, encoding: str) -> CaseInsensitiveDict: return CaseInsensitiveDict( {k.decode(encoding): v.decode(encoding) for k, v in headers.items()})
class App: app_config: AppConfig = None """ The contents of asyncy.yaml. """ config: Config = None """ The runtime config for this app. """ def __init__(self, app_data: AppData): self._subscriptions = {} self.app_id = app_data.app_id self.app_name = app_data.app_name self.app_dns = app_data.app_dns self.config = app_data.config self.app_config = app_data.app_config self.version = app_data.version self.logger = app_data.logger self.owner_uuid = app_data.owner_uuid self.owner_email = app_data.owner_email self.environment = app_data.environment if app_data.environment is None: self.environment = {} else: self.environment = app_data.environment self.environment = CaseInsensitiveDict(data=self.environment) self.stories = app_data.stories['stories'] self.entrypoint = app_data.stories['entrypoint'] self.services = app_data.services secrets = CaseInsensitiveDict() for k, v in self.environment.items(): if not isinstance(v, dict): secrets[k] = v self.app_context = { 'secrets': secrets, 'hostname': f'{self.app_dns}.{self.config.APP_DOMAIN}', 'version': self.version } async def bootstrap(self): """ Executes all stories found in stories.json. This enables the story to listen to pub/sub, register with the gateway, and queue cron jobs. """ await self.start_services() await self.expose_services() await self.run_stories() async def expose_services(self): for expose in self.app_config.get_expose_config(): await self._expose_service(expose) async def _expose_service(self, e: Expose): self.logger.info(f'Exposing service {e.service}/' f'{e.service_expose_name} ' f'on {e.http_path}') conf = Dict.find( self.services, f'{e.service}' f'.{ServiceConstants.config}' f'.expose.{e.service_expose_name}') if conf is None: raise StoryscriptError( message=f'Configuration for expose "{e.service_expose_name}" ' f'not found in service "{e.service}"') target_path = Dict.find(conf, 'http.path') target_port = Dict.find(conf, 'http.port') if target_path is None or target_port is None: raise StoryscriptError( message=f'http.path or http.port is null ' f'for expose {e.service}/{e.service_expose_name}') await Containers.expose_service(self, e) async def start_services(self): tasks = [] reusable_services = set() for story_name in self.stories.keys(): story = Stories(self, story_name, self.logger) line = story.first_line() while line is not None: line = story.line(line) method = line['method'] try: if method != 'execute': continue chain = Services.resolve_chain(story, line) assert isinstance(chain[0], Service) assert isinstance(chain[1], Command) if Containers.is_service_reusable(story.app, line): # Simple cache to not unnecessarily make more calls to # Kubernetes. It's okay if we don't have this check # though, since the underlying API handles this. service = chain[0].name if service in reusable_services: continue reusable_services.add(service) if not Services.is_internal(chain[0].name, chain[1].name): tasks.append(Services.start_container(story, line)) finally: line = line.get('next') if len(tasks) > 0: completed, pending = await asyncio.wait(tasks) # Pending must never be greater than zero. assert len(pending) == 0 for task in completed: exc = task.exception() if exc is not None: raise exc async def run_stories(self): """ Executes all the stories. This enables the story to listen to pub/sub, register with the gateway, and queue cron jobs. """ for story_name in self.entrypoint: await Story.run(self, self.logger, story_name) def add_subscription(self, sub_id: str, streaming_service: StreamingService, event: str, payload: dict): self._subscriptions[sub_id] = Subscription(streaming_service, sub_id, payload, event) def get_subscription(self, sub_id: str): return self._subscriptions.get(sub_id) def remove_subscription(self, sub_id: str): self._subscriptions.pop(sub_id) async def clear_subscriptions_synapse(self): url = f'http://{self.config.ASYNCY_SYNAPSE_HOST}:' \ f'{self.config.ASYNCY_SYNAPSE_PORT}/clear_all' kwargs = { 'method': 'POST', 'body': json.dumps({'app_id': self.app_id}), 'headers': { 'Content-Type': 'application/json; charset=utf-8' } } client = AsyncHTTPClient() response = await HttpUtils.fetch_with_retry(3, self.logger, url, client, kwargs) if int(response.code / 100) == 2: self.logger.debug(f'Unsubscribed all with Synapse!') return True else: self.logger.error(f'Failed to unsubscribe with Synapse!') return False async def unsubscribe_all(self): for sub_id, sub in self._subscriptions.items(): assert isinstance(sub, Subscription) assert isinstance(sub.streaming_service, StreamingService) conf = Dict.find( self.services, f'{sub.streaming_service.name}' f'.{ServiceConstants.config}' f'.actions.{sub.streaming_service.command}' f'.events.{sub.event}.http') http_conf = conf.get('unsubscribe') if not http_conf: self.logger.debug(f'No unsubscribe call required for {sub}') continue url = f'http://{sub.streaming_service.hostname}' \ f':{http_conf.get("port", conf.get("port", 80))}' \ f'{http_conf["path"]}' client = AsyncHTTPClient() self.logger.debug(f'Unsubscribing {sub}...') method = http_conf.get('method', 'post') kwargs = { 'method': method.upper(), 'body': json.dumps(sub.payload['sub_body']), 'headers': { 'Content-Type': 'application/json; charset=utf-8' } } response = await HttpUtils.fetch_with_retry( 3, self.logger, url, client, kwargs) if int(response.code / 100) == 2: self.logger.debug(f'Unsubscribed!') else: self.logger.error(f'Failed to unsubscribe {sub}!') async def destroy(self): """ Unsubscribe from all existing subscriptions, and delete the namespace. """ await self.clear_subscriptions_synapse() await self.unsubscribe_all()
class Drheader: """ Core functionality for DrHeader. This is where the magic happens """ error_types = { 1: 'Header not included in response', 2: 'Header should not be returned', 3: 'Value does not match security policy', 4: 'Must-Contain directive missed', 5: 'Must-Avoid directive included', 6: 'Must-Contain-One directive missed', 7: 'Directive not included in response', 8: 'Directive should not be returned' } def __init__(self, url=None, method="GET", headers=None, status_code=None, params=None, request_headers=None, verify=True): """ NOTE: at least one param required. :param url: (optional) URL of target :type url: str :param method: (optional) Method to use when doing the request :type method: str :param headers: (optional) Override headers :type headers: dict :param status_code: Override status code :type status_code: int :param params: Request params :type params: dict :param request_headers: Request headers :type request_headers: dict :param verify: Verify the server's TLS certificate :type verify: bool or str """ if request_headers is None: request_headers = {} if isinstance(headers, str): headers = json.loads(headers) elif url and not headers: headers, status_code = self._get_headers(url, method, params, request_headers, verify) self.status_code = status_code self.headers = CaseInsensitiveDict(headers) self.anomalies = [] self.url = url self.delimiter = ';' self.report = [] @staticmethod def _get_headers(url, method, params, request_headers, verify): """ Get headers for specified url. :param url: URL of target :type url: str :param method: (optional) Method to use when doing the request :type method: str :param params: Request params :type params: dict :param request_headers: Request headers :type request_headers: dict :param verify: Verify the server's TLS certificate :type verify: bool or str :return: headers, status_code :rtype: dict, int """ if validators.url(url): req_obj = getattr(requests, method.lower()) r = req_obj(url, data=params, headers=request_headers, verify=verify) headers = r.headers if len(r.raw.headers.getlist('Set-Cookie')) > 0: headers['set-cookie'] = r.raw.headers.getlist('Set-Cookie') return headers, r.status_code def analyze(self, rules=None): """ Analyze the currently loaded headers against provided rules. :param rules: Override rules to compare headers against :type rules: dict :return: Audit report :rtype: list """ for header, value in self.headers.items(): if type(value) == str: self.headers[header] = value.lower() if type(value) == list: value = [item.lower() for item in value] self.headers[header] = value if not rules: rules = load_rules() for rule, config in rules.items(): self.__validate_rules(config, header=rule) if 'Directives' in config and rule in self.headers: for directive, d_config in config['Directives'].items(): self.__validate_rules(d_config, header=rule, directive=directive) return self.report def __validate_rule_and_value(self, expected_value, header, directive): """ Verify headers content matches provided config. :param expected_value: Expected value of header. :param header: Name of header :param directive: Name of directive (optional) :return: """ expected_value_list = [str(item).lower() for item in expected_value] if len(expected_value_list) == 1: expected_value_list = [ item.strip(' ') for item in expected_value_list[0].split(self.delimiter) ] if directive: rule = directive headers = _to_dict(self.headers[header], ';', ' ') else: rule = header headers = self.headers if rule not in headers: self.__add_report_item(severity='high', error_type=7 if directive else 1, header=header, directive=directive, expected=expected_value_list) else: rule_list = [ item.strip(' ') for item in headers[rule].split(self.delimiter) ] if not all(elem in expected_value_list for elem in rule_list): self.__add_report_item(severity='high', error_type=3, header=header, directive=directive, expected=expected_value_list, value=headers[rule]) def __validate_not_exists(self, header, directive): """ Verify specified rule does not exist in loaded headers. :param header: Name of header :param directive: Name of directive (optional) """ if directive: rule = directive headers = _to_dict(self.headers[header], ';', ' ') else: rule = header headers = self.headers if rule in headers: self.__add_report_item(severity='high', error_type=8 if directive else 2, header=header, directive=directive) def __validate_exists(self, header, directive): """ Verify specified rule exists in loaded headers. :param header: Name of header :param directive: Name of directive (optional) """ if directive: rule = directive headers = _to_dict(self.headers[header], ';', ' ') else: rule = header headers = self.headers if rule not in headers: self.__add_report_item(severity='high', error_type=7 if directive else 1, header=header, directive=directive) return rule in headers # Return value to prevent subsequent avoid/contain checks if the header is not present def __validate_must_avoid(self, config, header, directive): """ Verify specified values do not exist in loaded headers. :param config: Configuration rule-set to use :param header: Name of header :param directive: Name of directive (optional) """ if directive: rule = directive header_value = _to_dict(self.headers[header], ';', ' ')[rule] else: rule = header header_value = self.headers[rule] config['Must-Avoid'] = [item.lower() for item in config['Must-Avoid']] for avoid_value in config['Must-Avoid']: if avoid_value in header_value and rule not in self.anomalies: if rule.lower() == 'content-security-policy': policy = _to_dict(self.headers[header], ';', ' ') non_compliant_values = [ item for item in list(policy.values()) if avoid_value in item ] indices = [ list(policy.values()).index(item) for item in non_compliant_values ] for index in indices: self.__add_report_item(severity='medium', error_type=5, header=header, directive=list( policy.keys())[index], avoid=config['Must-Avoid'], value=avoid_value) else: self.__add_report_item(severity='medium', error_type=5, header=header, directive=directive, avoid=config['Must-Avoid'], value=avoid_value) def __validate_must_contain(self, config, header, directive): """ Verify the provided header contains certain params. :param config: Configuration rule-set to use :param header: Name of header :param directive: Name of directive (optional) """ if directive: rule = directive header_value = _to_dict(self.headers[header], ';', ' ')[rule] else: rule = header header_value = self.headers[rule] if 'Must-Contain-One' in config: config['Must-Contain-One'] = [ item.lower() for item in config['Must-Contain-One'] ] contain_values = header_value.split( ' ') if directive else header_value.split(self.delimiter) does_contain = False for contain_value in contain_values: contain_value = contain_value.lstrip() if contain_value in config['Must-Contain-One']: does_contain = True break if not does_contain: self.__add_report_item(severity='high', error_type=6, header=header, directive=directive, expected=config['Must-Contain-One'], value=config['Must-Contain-One']) elif 'Must-Contain' in config: config['Must-Contain'] = [ item.lower() for item in config['Must-Contain'] ] if header.lower() == 'set-cookie': for cookie in self.headers[header]: for contain_value in config['Must-Contain']: if contain_value not in cookie: self.__add_report_item( severity='high' if contain_value == 'secure' else 'medium', error_type=4, header=header, expected=config['Must-Contain'], value=contain_value, cookie=cookie) else: for contain_value in config['Must-Contain']: if contain_value not in header_value and rule not in self.anomalies: self.__add_report_item(severity='medium', error_type=4, header=header, directive=directive, expected=config['Must-Contain'], value=contain_value) def __validate_rules(self, config, header, directive=None): """ Entry point for validation. :param config: Configuration rule-set to use :param header: Name of header :param directive: Name of directive (optional) """ try: self.delimiter = config['Delimiter'] except KeyError: self.delimiter = ';' if config['Required'] is True or (config['Required'] == 'Optional' and header in self.headers): if config['Enforce']: self.__validate_rule_and_value(config['Value'], header, directive) else: exists = self.__validate_exists(header, directive) if exists: if 'Must-Contain-One' in config or 'Must-Contain' in config: self.__validate_must_contain(config, header, directive) if 'Must-Avoid' in config: self.__validate_must_avoid(config, header, directive) elif config['Required'] is False: self.__validate_not_exists(header, directive) def __add_report_item(self, severity, error_type, header, directive=None, expected=None, avoid=None, value='', cookie=''): """ Add a entry to report. :param severity: [low, medium, high] :type severity: str :param error_type: [1...6] related to error_types :type error_type: int :param expected: Expected value of header :param avoid: Avoid value of header :param value: Current value of header :param cookie: Value of cookie (if applicable) """ if directive: error = { 'rule': header + ' - ' + directive, 'severity': severity, 'message': self.error_types[error_type] } else: error = { 'rule': header, 'severity': severity, 'message': self.error_types[error_type] } if expected: error['expected'] = expected error['delimiter'] = self.delimiter if avoid: error['avoid'] = avoid error['delimiter'] = self.delimiter if error_type == 3: error['value'] = value elif error_type in (4, 5, 6): if header.lower() == 'set-cookie': error['value'] = cookie else: if directive: error['value'] = _to_dict(self.headers[header], ';', ' ')[directive].strip('\'') else: error['value'] = self.headers[header] error['anomaly'] = value self.report.append(error)
class Request: def __init__(self, method, url, data=None, headers=None): """Request objects :param method: an HTTP method :param url: a full URL including query string if needed :param data: data to be sent in the body of POST request :param headers: dict representing headers. Case-insensitive Content-Type header should be given at least if data exists """ self.method = method.upper() self.url = url if self.method == 'GET': self.data = None elif self.method == 'POST': self.data = data else: raise NotImplementedError('{}: unknown method'.format(self.method)) r = urlparse(self.url) if r.scheme not in ('http', ): raise NotImplementedError('{}: not supported'.format(r.scheme)) self.hostname = r.hostname self.port = r.port if r.port else 80 # set request path self.path = r.path or '/' # append params and query to path if exist if r.params: self.path += ';' + r.params if r.query: self.path += '?' + r.query # make headers self.headers = CaseInsensitiveDict() self.headers['Host'] = self.hostname self.headers['Agent'] = 'httpcli' self.headers['Accept'] = '*/*' self.headers['Connection'] = 'keep-alive' if headers: for key, value in headers.items(): self.headers[key] = value if self.data: # for PUT method if not self.headers.get('content-type', None): raise ValueError('No Content-type header specified') self.headers['content-length'] = str(len(self.data)) def open(self): """Open an HTTP session. Then, send/receive request/response. """ self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.sock.connect((self.hostname, self.port)) self.file = file = self.sock.makefile('rwb') headers_list = [ key + ': ' + value for key, value in self.headers.items() ] headers_str = '\r\n'.join(headers_list) + '\r\n' template = '{method} {path} HTTP/1.1\r\n{headers}\r\n' message = template.format(method=self.method, path=self.path, headers=headers_str) logging.debug('Request message:\n{}'.format(message)) file.write(message.encode('utf-8')) if self.data: file.write(self.data.encode('utf-8')) file.flush() response = Response(self) # read status line response.status_code = response.read_status() logging.debug('status code: {}'.format(response.status_code)) # read headers response.read_headers() logging.debug('Response headers:\n{}'.format(response.headers)) # read contents response.read_content() if response.status_code >= 400: # HTTP error response logging.warning(response.content) return response def close(self): """Close this HTTP session """ if not self.sock._closed: self.file.close() self.sock.close()