def __init__(self, service, table, address, db): """ :param table: ClickHouse table name :param address: ClickHouse address :param db: ClickHouse database :return: """ self.name = table self.service = service self.address = address self.db = db self.sql = "INSERT INTO %s FORMAT JSONEachRow" % table self.encoded_sql = urllib_quote(self.sql.encode("utf8")) self.n = 0 self.data = [] self.last_updated = perf_counter() self.last_flushed = perf_counter() self.flushing = False self.url = "http://%s/?user=%s&password=%s&database=%s&query=%s" % ( address, config.clickhouse.rw_user, config.clickhouse.rw_password or "", db, self.encoded_sql, )
def execute(self, sql=None, args=None, nodb=False, post=None, extra=None): def q(v): # @todo: quote dates if isinstance(v, six.string_types): return "'%s'" % (v.replace("\\", "\\\\").replace("'", "\\'")) else: return str(v) qs = [] if not nodb: qs += ["database=%s" % config.clickhouse.db] if extra: qs += ["%s=%s" % (k, v) for k, v in extra] if sql: if args: sql = sql % tuple(q(v) for v in args) if post: qs += ["query=%s" % urllib_quote(sql.encode("utf8"))] else: post = sql.encode("utf8") url = "http://%s/?%s" % (random.choice(self.addresses), "&".join(qs)) code, headers, body = fetch_sync( url, method="POST", body=post, user=self.user, password=self.password, connect_timeout=config.clickhouse.connect_timeout, request_timeout=config.clickhouse.request_timeout, ) if code != 200: raise ClickhouseError("%s: %s" % (code, body)) return [row.split("\t") for row in body.splitlines()]
def _get_authorization_token(account_key, resource_link, resource_type, verb, date): ''' Returns the authorization header. :param str account_key: The account key. :param str resource_link: Identity property of the resource that the request is directed at, eg. "dbs/MyDatabase/colls/MyCollection". :param str resource_type: Type of resource that the request is for, eg. "dbs", "colls", "docs". :param str verb: HTTP verb, such as GET, POST or PUT. :param str date: String of UTC date and time in HTTP-date format. This same date (in same format) also needs to be passed as x-ms-date header in the request. ''' string_to_sign = '{}\n{}\n{}\n{}\n\n'.format(_lower(verb), _lower(resource_type), resource_link, _lower(date)) signature = _sign_string(account_key, string_to_sign) return urllib_quote('type=master&ver=1.0&sig={}'.format(signature), '-_.!~*\'()')
def _get_char_id_ps(self, font, ccode): """ Return a unique id for the given font and character-code set (for tex). """ ps_name = font.get_ps_font_info()[2] char_id = urllib_quote('%s-%d' % (ps_name, ccode)) return char_id
def __init__(self, service, fields, address, db): """ :param fields: <table>.<field1>. .. .<fieldN> :return: """ self.name = fields self.service = service self.address = address self.db = db if "|" in fields: # New format. Separated by '|'. # Nested fields are possible parts = tuple(fields.split("|")) else: # Old format. Separated by '.'. # Nested fields are not possible parts = tuple(fields.split(".")) self.sql = "INSERT INTO %s(%s) FORMAT TabSeparated" % ( parts[0], ",".join(parts[1:])) self.encoded_sql = urllib_quote(self.sql.encode("utf8")) self.n = 0 self.data = [] self.last_updated = perf_counter() self.last_flushed = perf_counter() self.flushing = False self.url = "http://%s/?user=%s&password=%s&database=%s&query=%s" % ( address, config.clickhouse.rw_user, config.clickhouse.rw_password or "", db, self.encoded_sql, )
def _api_call(url, type, auth_args, http_timeout): """ Makes a REST call against the Jenkins API. Args: url (str): The URL to get, including endpoint Returns: list: The JSON response """ parsed_url = urllib.parse.urlparse(url) url = "{0}://{1}{2}".format(parsed_url.scheme, parsed_url.netloc, urllib_quote(parsed_url.path)) if parsed_url.query: url = url + "?" + parsed_url.query resp = None try: resp = requests.get(url, timeout=http_timeout, **auth_args) resp.raise_for_status() return load_json(resp.text, url) except HTTPError as e: if e.response.status_code == 500 and type == "healthcheck": return load_json(e.response.text, url) else: collectd.error("Error making API call (%s) %s" % (e, url)) return None except RequestException as e: collectd.error("Error making API call (%s) %s" % (e, url)) return None finally: if resp: resp.close()
def mongo_connection_args(self): """ Mongo connection arguments. Suitable to pass to pymongo.connect and mongoengine.connect """ if not hasattr(self, "_mongo_connection_args"): self._mongo_connection_args = { "db": self.mongo.db, "username": self.mongo.user, "password": self.mongo.password, } if self.mongo.app_name: self._mongo_connection_args["appname"] = self.mongo.app_name if self.mongo.retry_writes: self._mongo_connection_args["retryWrites"] = True has_credentials = self.mongo.user or self.mongo.password if has_credentials: self._mongo_connection_args[ "authentication_source"] = self.mongo.db hosts = self.mongo.addresses if self.mongo.rs: self._mongo_connection_args["replicaSet"] = self.mongo.rs self._mongo_connection_args[ "readPreference"] = "secondaryPreferred" elif len(hosts) > 1: raise ValueError("Replica set name must be set") if self.mongo.max_idle_time: self._mongo_connection_args[ "maxIdleTimeMS"] = self.mongo.max_idle_time * 1000 url = ["mongodb://"] if has_credentials: url += [ "%s:%s@" % (urllib_quote( self.mongo.user), urllib_quote(self.mongo.password)) ] url += [",".join(str(h) for h in hosts)] url += ["/%s" % self.mongo.db] self._mongo_connection_args["host"] = "".join(url) if self.metrics.enable_mongo_hist: from noc.core.mongo.monitor import MongoCommandSpan self._mongo_connection_args["event_listeners"] = [ MongoCommandSpan() ] return self._mongo_connection_args
def forward(self, query, bounds=None, region=None): query = query.lower().strip() if not query: return None url = ["http://maps.googleapis.com/maps/api/geocode/json?"] if region: url += ["®ion=%s" % region] if bounds: # &bounds=34.172684,-118.604794|34.236144,-118.500938 # bounds = ("34.172684,-118.604794", "34.236144,-118.500938") url += ["&bounds=%s|%s" % bounds] url += ["&address=%s" % urllib_quote(query)] if self.key: url += ["&key=%s" % urllib_quote(self.key)] if self.language: url += ["&language=%s" % urllib_quote(self.language)] code, response = self.get("".join(url)) if code != 200: raise GeoCoderError("%s: %s" % (code, response)) try: r = ujson.loads(response) except ValueError: raise GeoCoderError("Cannot decode result") if r["status"] != "OK": return None for rr in r["results"]: lon = self.get_path(rr, "geometry.location.lng") lat = self.get_path(rr, "geometry.location.lat") if not rr.get("address_components"): return None path = [x["short_name"] for x in rr["address_components"]] if "postal_code" in rr["address_components"][-1]["types"]: path = path[:-1] path.reverse() is_exact = (self.get_path( rr, "GeoObject.metaDataProperty.GeocoderMetaData.precision") == "exact") return GeoCoderResult(exact=is_exact, query=query, path=path, lon=lon, lat=lat)
def _get_char_id(self, font, ccode): """ Return a unique id for the given font and character-code set. """ sfnt = font.get_sfnt() try: ps_name = sfnt[1, 0, 0, 6].decode('mac_roman') except KeyError: ps_name = sfnt[3, 1, 0x0409, 6].decode('utf-16be') char_id = urllib_quote('%s-%x' % (ps_name, ccode)) return char_id
def forward(self, s): url = "http://nominatim.openstreetmap.org/search?q=" url += urllib_quote(s) + "&format=json&addressdetails=1" r = requests.get(url) d = r.json() if d: lon = float(d[0]["lon"]) lat = float(d[0]["lat"]) return "EPSG:4326", lon, lat else: return None, None, None
def _get_char_id(self, font, ccode): """ Return a unique id for the given font and character-code set. """ sfnt = font.get_sfnt() try: ps_name = sfnt[(1, 0, 0, 6)].decode('macroman') except KeyError: ps_name = sfnt[(3, 1, 0x0409, 6)].decode('utf-16be') char_id = urllib_quote('%s-%x' % (ps_name, ccode)) return char_id
def forward(self, query, bounds=None, region=None): url = ( "http://nominatim.openstreetmap.org/search?q=%s&format=json&addressdetails=1" % urllib_quote(query) ) if bounds: url += "viewbox=%s,%s" % bounds code, response = self.get("".join(url)) if code != 200: raise GeoCoderError("%s: %s" % (code, response)) try: r = ujson.loads(response) except ValueError: raise GeoCoderError("Cannot decode result") return GeoCoderResult(exact=False, query=query, path=[], lon=r[0]["lon"], lat=r[0]["lat"])
def GetDocumentCollectionInfo(self_link, alt_content_path, id_from_response): """ Given the self link and alt_content_path from the reponse header and result extract the collection name and collection id Ever response header has alt-content-path that is the owner's path in ascii. For document create / update requests, this can be used to get the collection name, but for collection create response, we can't use it. So we also rely on :param str self_link: Self link of the resource, as obtained from response result. :param str alt_content_path: Owner path of the resource, as obtained from response header. :param str resource_id: 'id' as returned from the response result. This is only used if it is deduced that the request was to create a collection. :return: tuple of (collection rid, collection name) :rtype: tuple """ self_link = TrimBeginningAndEndingSlashes(self_link) + '/' index = IndexOfNth(self_link, '/', 4) if index != -1: collection_id = self_link[0:index] if 'colls' in self_link: # this is a collection request index_second_slash = IndexOfNth(alt_content_path, '/', 2) if index_second_slash == -1: collection_name = alt_content_path + '/colls/' + urllib_quote( id_from_response) return collection_id, collection_name else: collection_name = alt_content_path return collection_id, collection_name else: raise ValueError( 'Response Not from Server Partition, self_link: {0}, alt_content_path: {1},' + 'id: {2}'.format(self_link, alt_content_path, id_from_response) ) else: raise ValueError('Unable to parse document collection link from ' + self_link)
def get_authorization(client, verb, resource_id_or_fullname, resource_type, headers): authorization = auth.GetAuthorizationHeader( cosmos_client_connection=client, verb=verb, path='', resource_id_or_fullname=resource_id_or_fullname, is_name_based=True, resource_type=resource_type, headers=headers) # urllib.quote throws when the input parameter is None if authorization: # -_.!~*'() are valid characters in url, and shouldn't be quoted. authorization = urllib_quote(authorization, '-_.!~*\'()') return authorization
def searchLocation(self): if self.current.city.value != "": language = config.osd.language.value.replace("_", "-") if language == "en-EN": # hack language = "en-US" elif language == "no-NO": # hack language = "nn-NO" url = "http://weather.service.msn.com/find.aspx?src=windows&outputview=search&weasearchstr=%s&culture=%s" % ( urllib_quote(self.current.city.value), language) getPage(six.ensure_binary(url)).addCallback( self.xmlCallback).addErrback(self.error) else: self.session.open( MessageBox, _("You need to enter a valid city name before you can search for the location code." ), MessageBox.TYPE_ERROR)
def GetItemContainerInfo(self_link, alt_content_path, id_from_response): """ Given the self link and alt_content_path from the reponse header and result extract the collection name and collection id Ever response header has alt-content-path that is the owner's path in ascii. For document create / update requests, this can be used to get the collection name, but for collection create response, we can't use it. So we also rely on :param str self_link: Self link of the resource, as obtained from response result. :param str alt_content_path: Owner path of the resource, as obtained from response header. :param str resource_id: 'id' as returned from the response result. This is only used if it is deduced that the request was to create a collection. :return: tuple of (collection rid, collection name) :rtype: tuple """ self_link = TrimBeginningAndEndingSlashes(self_link) + '/' index = IndexOfNth(self_link, '/', 4) if index != -1: collection_id = self_link[0:index] if 'colls' in self_link: # this is a collection request index_second_slash = IndexOfNth(alt_content_path, '/', 2) if index_second_slash == -1: collection_name = alt_content_path + '/colls/' + urllib_quote(id_from_response) return collection_id, collection_name else: collection_name = alt_content_path return collection_id, collection_name else: raise ValueError('Response Not from Server Partition, self_link: {0}, alt_content_path: {1}, id: {2}' .format(self_link, alt_content_path, id_from_response)) else: raise ValueError('Unable to parse document collection link from ' + self_link)
def GetItemContainerInfo(self_link, alt_content_path, id_from_response): """Given the self link and alt_content_path from the reponse header and result extract the collection name and collection id. Every response header has an alt-content-path that is the owner's path in ASCII. For document create / update requests, this can be used to get the collection name, but for collection create response, we can't use it. :param str self_link: Self link of the resource, as obtained from response result. :param str alt_content_path: Owner path of the resource, as obtained from response header. :param str resource_id: 'id' as returned from the response result. This is only used if it is deduced that the request was to create a collection. :return: tuple of (collection rid, collection name) :rtype: tuple """ self_link = TrimBeginningAndEndingSlashes(self_link) + "/" index = IndexOfNth(self_link, "/", 4) if index != -1: collection_id = self_link[0:index] if "colls" in self_link: # this is a collection request index_second_slash = IndexOfNth(alt_content_path, "/", 2) if index_second_slash == -1: collection_name = alt_content_path + "/colls/" + urllib_quote(id_from_response) return collection_id, collection_name collection_name = alt_content_path return collection_id, collection_name raise ValueError( "Response Not from Server Partition, self_link: {0}, alt_content_path: {1}, id: {2}".format( self_link, alt_content_path, id_from_response ) ) raise ValueError("Unable to parse document collection link from " + self_link)
def GetPathFromLink(resource_link, resource_type=""): """Gets path from resource link with optional resource type :param str resource_link: :param str resource_type: :return: Path from resource link with resource type appended (if provided). :rtype: str """ resource_link = TrimBeginningAndEndingSlashes(resource_link) if IsNameBased(resource_link): # Replace special characters in string using the %xx escape. For example, # space(' ') would be replaced by %20 This function is intended for quoting # the path section of the URL and excludes '/' to be quoted as that's the # default safe char resource_link = urllib_quote(resource_link) # Padding leading and trailing slashes to the path returned both for name based and resource id based links if resource_type: return "/" + resource_link + "/" + resource_type + "/" return "/" + resource_link + "/"
def getWeatherData(self, degreetype, locationcode, city, callback, callbackShowIcon, callbackAllIconsDownloaded=None): self.initialize() language = config.osd.language.value.replace("_", "-") if language == "en-EN": # hack language = "en-US" elif language == "no-NO": # hack language = "nn-NO" self.city = city self.callback = callback self.callbackShowIcon = callbackShowIcon self.callbackAllIconsDownloaded = callbackAllIconsDownloaded url = "http://weather.service.msn.com/data.aspx?src=windows&weadegreetype=%s&culture=%s&wealocations=%s" % ( degreetype, language, urllib_quote(locationcode)) getPage(six.ensure_binary(url)).addCallback( self.xmlCallback).addErrback(self.error)
def GetPathFromLink(resource_link, resource_type=''): """Gets path from resource link with optional resource type :param str resource_link: :param str resource_type: :return: Path from resource link with resource type appended (if provided). :rtype: str """ resource_link = TrimBeginningAndEndingSlashes(resource_link) if IsNameBased(resource_link): # Replace special characters in string using the %xx escape. For example, space(' ') would be replaced by %20 # This function is intended for quoting the path section of the URL and excludes '/' to be quoted as that's the default safe char resource_link = urllib_quote(resource_link) # Padding leading and trailing slashes to the path returned both for name based and resource id based links if resource_type: return '/' + resource_link + '/' + resource_type + '/' else: return '/' + resource_link + '/'
def GetPathFromLink(resource_link, resource_type=''): """Gets path from resource link with optional resource type :Parameters: - `resource_link`: str - `resource_type`: str :Returns: str, path from resource link with resource type appended(if provided). """ resource_link = TrimBeginningAndEndingSlashes(resource_link) if IsNameBased(resource_link): # Replace special characters in string using the %xx escape. For example, space(' ') would be replaced by %20 # This function is intended for quoting the path section of the URL and excludes '/' to be quoted as that's the default safe char resource_link = urllib_quote(resource_link) # Padding leading and trailing slashes to the path returned both for name based and resource id based links if resource_type: return '/' + resource_link + '/' + resource_type + '/' else: return '/' + resource_link + '/'
def search(self, callback, searchkey, _page=1, _per_page=30): """wct.search.webcams Search the webcams by the given query. Arguments devid (required) Your developer ID. If you do not have one, please signup for a developer ID. query (required) The query to search for. per_page (optional) Number of comments to return per page. If this argument is omitted, it defaults to 10. The maximum allowed value is 50. page (optional) The page of results to return. If this argument is omitted, it defaults to 1. """ cb = lambda raw: self.searchCB(raw, callback) self.get("wct.search.webcams", cb, None, query=urllib_quote(searchkey), page=_page, per_page=_per_page)
def GetHeaders( # pylint: disable=too-many-statements,too-many-branches cosmos_client_connection, default_headers, verb, path, resource_id, resource_type, options, partition_key_range_id=None, ): """Gets HTTP request headers. :param cosmos_client_connection.CosmosClient cosmos_client: :param dict default_headers: :param str verb: :param str path: :param str resource_id: :param str resource_type: :param dict options: :param str partition_key_range_id: :return: The HTTP request headers. :rtype: dict """ headers = dict(default_headers) options = options or {} if cosmos_client_connection._useMultipleWriteLocations: headers[http_constants.HttpHeaders.AllowTentativeWrites] = "true" pre_trigger_include = options.get("preTriggerInclude") if pre_trigger_include: headers[http_constants.HttpHeaders.PreTriggerInclude] = ( pre_trigger_include if isinstance(pre_trigger_include, str) else (",").join(pre_trigger_include)) post_trigger_include = options.get("postTriggerInclude") if post_trigger_include: headers[http_constants.HttpHeaders.PostTriggerInclude] = ( post_trigger_include if isinstance(post_trigger_include, str) else (",").join(post_trigger_include)) if options.get("maxItemCount"): headers[http_constants.HttpHeaders.PageSize] = options["maxItemCount"] access_condition = options.get("accessCondition") if access_condition: if access_condition["type"] == "IfMatch": headers[http_constants.HttpHeaders. IfMatch] = access_condition["condition"] else: headers[http_constants.HttpHeaders. IfNoneMatch] = access_condition["condition"] if options.get("indexingDirective"): headers[http_constants.HttpHeaders. IndexingDirective] = options["indexingDirective"] consistency_level = None # get default client consistency level default_client_consistency_level = headers.get( http_constants.HttpHeaders.ConsistencyLevel) # set consistency level. check if set via options, this will override the default if options.get("consistencyLevel"): consistency_level = options["consistencyLevel"] headers[ http_constants.HttpHeaders.ConsistencyLevel] = consistency_level elif default_client_consistency_level is not None: consistency_level = default_client_consistency_level headers[ http_constants.HttpHeaders.ConsistencyLevel] = consistency_level # figure out if consistency level for this request is session is_session_consistency = consistency_level == documents.ConsistencyLevel.Session # set session token if required if is_session_consistency is True and not IsMasterResource(resource_type): # if there is a token set via option, then use it to override default if options.get("sessionToken"): headers[http_constants.HttpHeaders. SessionToken] = options["sessionToken"] else: # check if the client's default consistency is session (and request consistency level is same), # then update from session container if default_client_consistency_level == documents.ConsistencyLevel.Session: # populate session token from the client's session container headers[ http_constants.HttpHeaders. SessionToken] = cosmos_client_connection.session.get_session_token( path) if options.get("enableScanInQuery"): headers[http_constants.HttpHeaders. EnableScanInQuery] = options["enableScanInQuery"] if options.get("resourceTokenExpirySeconds"): headers[http_constants.HttpHeaders. ResourceTokenExpiry] = options["resourceTokenExpirySeconds"] if options.get("offerType"): headers[http_constants.HttpHeaders.OfferType] = options["offerType"] if options.get("offerThroughput"): headers[http_constants.HttpHeaders. OfferThroughput] = options["offerThroughput"] if "partitionKey" in options: # if partitionKey value is Undefined, serialize it as [{}] to be consistent with other SDKs. if options.get("partitionKey") is partition_key._Undefined: headers[http_constants.HttpHeaders.PartitionKey] = [{}] # If partitionKey value is Empty, serialize it as [], which is the equivalent sent for migrated collections elif options.get("partitionKey") is partition_key._Empty: headers[http_constants.HttpHeaders.PartitionKey] = [] # else serialize using json dumps method which apart from regular values will serialize None into null else: headers[http_constants.HttpHeaders.PartitionKey] = json.dumps( [options["partitionKey"]]) if options.get("enableCrossPartitionQuery"): headers[ http_constants.HttpHeaders. EnableCrossPartitionQuery] = options["enableCrossPartitionQuery"] if options.get("populateQueryMetrics"): headers[http_constants.HttpHeaders. PopulateQueryMetrics] = options["populateQueryMetrics"] if cosmos_client_connection.master_key: headers[http_constants.HttpHeaders.XDate] = datetime.datetime.utcnow( ).strftime("%a, %d %b %Y %H:%M:%S GMT") if cosmos_client_connection.master_key or cosmos_client_connection.resource_tokens: authorization = auth.GetAuthorizationHeader(cosmos_client_connection, verb, path, resource_id, IsNameBased(resource_id), resource_type, headers) # urllib.quote throws when the input parameter is None if authorization: # -_.!~*'() are valid characters in url, and shouldn't be quoted. authorization = urllib_quote(authorization, "-_.!~*'()") headers[http_constants.HttpHeaders.Authorization] = authorization if verb in ("post", "put"): if not headers.get(http_constants.HttpHeaders.ContentType): headers[http_constants.HttpHeaders. ContentType] = _runtime_constants.MediaTypes.Json if not headers.get(http_constants.HttpHeaders.Accept): headers[http_constants.HttpHeaders. Accept] = _runtime_constants.MediaTypes.Json if partition_key_range_id is not None: headers[http_constants.HttpHeaders. PartitionKeyRangeID] = partition_key_range_id if options.get("enableScriptLogging"): headers[http_constants.HttpHeaders. EnableScriptLogging] = options["enableScriptLogging"] if options.get("offerEnableRUPerMinuteThroughput"): headers[http_constants.HttpHeaders. OfferIsRUPerMinuteThroughputEnabled] = options[ "offerEnableRUPerMinuteThroughput"] if options.get("disableRUPerMinuteUsage"): headers[http_constants.HttpHeaders. DisableRUPerMinuteUsage] = options["disableRUPerMinuteUsage"] if options.get("changeFeed") is True: # On REST level, change feed is using IfNoneMatch/ETag instead of continuation. if_none_match_value = None if options.get("continuation"): if_none_match_value = options["continuation"] elif options.get("isStartFromBeginning" ) and not options["isStartFromBeginning"]: if_none_match_value = "*" if if_none_match_value: headers[ http_constants.HttpHeaders.IfNoneMatch] = if_none_match_value headers[http_constants.HttpHeaders. AIM] = http_constants.HttpHeaders.IncrementalFeedHeaderValue else: if options.get("continuation"): headers[http_constants.HttpHeaders. Continuation] = options["continuation"] if options.get("populatePartitionKeyRangeStatistics"): headers[http_constants.HttpHeaders. PopulatePartitionKeyRangeStatistics] = options[ "populatePartitionKeyRangeStatistics"] if options.get("populateQuotaInfo"): headers[http_constants.HttpHeaders. PopulateQuotaInfo] = options["populateQuotaInfo"] return headers
def forward(self, query, bounds=None, region=None): url = ["https://geocode-maps.yandex.ru/1.x/?", "format=json"] if region: url += ["®ion=%s" % region] if bounds: # "&rspn=1&bbox=127.56,49.96~141.05,56.09" url += ["&rspn=1", "&bbox=%s~%s" % bounds] url += ["&geocode=%s" % urllib_quote(query)] if self.key: url += ["&key=%s" % urllib_quote(self.key)] if self.apikey: url += ["&apikey=%s" % urllib_quote(self.apikey)] code, response = self.get("".join(url)) if code == 429: raise GeoCoderLimitExceeded() elif code != 200: raise GeoCoderError("%s: %s" % (code, response)) try: r = ujson.loads(response) except ValueError: raise GeoCoderError("Cannot decode result") results = self.get_path( r, "response.GeoObjectCollection.featureMember") or [] for rr in results: pos = self.get_path(rr, "GeoObject.Point.pos") if pos: lon, lat = [float(x) for x in pos.split()] else: lon, lat = None, None path = [ self.get_path( rr, "GeoObject.metaDataProperty.GeocoderMetaData.AddressDetails.Country.CountryName", ), self.get_path( rr, "GeoObject.metaDataProperty.GeocoderMetaData.AddressDetails.Country.AdministrativeArea.AdministrativeAreaName", ), self.get_path( rr, "GeoObject.metaDataProperty.GeocoderMetaData.AddressDetails.Country.AdministrativeArea.SubAdministrativeArea.SubAdministrativeAreaName", ), self.get_path( rr, "GeoObject.metaDataProperty.GeocoderMetaData.AddressDetails.Country.AdministrativeArea.SubAdministrativeArea.Locality.LocalityName", ), self.get_path( rr, "GeoObject.metaDataProperty.GeocoderMetaData.AddressDetails.Country.AdministrativeArea.SubAdministrativeArea.Locality.Thoroughfare.ThoroughfareName", ), self.get_path( rr, "GeoObject.metaDataProperty.GeocoderMetaData.AddressDetails.Country.AdministrativeArea.SubAdministrativeArea.Locality.Thoroughfare.Premise.PremiseNumber", ), ] path = [p for p in path if p] is_exact = (self.get_path( rr, "GeoObject.metaDataProperty.GeocoderMetaData.precision") == "exact") return GeoCoderResult(exact=is_exact, query=query, path=path, lon=lon, lat=lat)
def GetHeaders(document_client, default_headers, verb, path, resource_id, resource_type, options, partition_key_range_id = None): """Gets HTTP request headers. :Parameters: - `document_client`: document_client.DocumentClient - `default_headers`: dict - `verb`: str - `path`: str - `resource_id`: str - `resource_type`: str - `options`: dict - `partition_key_range_id` : str :Returns: dict, the HTTP request headers. """ headers = dict(default_headers) options = options or {} if options.get('continuation'): headers[http_constants.HttpHeaders.Continuation] = ( options['continuation']) pre_trigger_include = options.get('preTriggerInclude') if pre_trigger_include: headers[http_constants.HttpHeaders.PreTriggerInclude] = ( pre_trigger_include if isinstance(pre_trigger_include, str) else (',').join(pre_trigger_include)) post_trigger_include = options.get('postTriggerInclude') if post_trigger_include: headers[http_constants.HttpHeaders.PostTriggerInclude] = ( post_trigger_include if isinstance(post_trigger_include, str) else (',').join(post_trigger_include)) if options.get('maxItemCount'): headers[http_constants.HttpHeaders.PageSize] = options['maxItemCount'] access_condition = options.get('accessCondition') if access_condition: if access_condition['type'] == 'IfMatch': headers[http_constants.HttpHeaders.IfMatch] = access_condition['condition'] else: headers[http_constants.HttpHeaders.IfNoneMatch] = access_condition['condition'] if options.get('indexingDirective'): headers[http_constants.HttpHeaders.IndexingDirective] = ( options['indexingDirective']) consistency_level = None ''' get default client consistency level''' default_client_consistency_level = headers.get(http_constants.HttpHeaders.ConsistencyLevel) ''' set consistency level. check if set via options, this will override the default ''' if options.get('consistencyLevel'): consistency_level = options['consistencyLevel'] headers[http_constants.HttpHeaders.ConsistencyLevel] = consistency_level elif default_client_consistency_level is not None: consistency_level = default_client_consistency_level headers[http_constants.HttpHeaders.ConsistencyLevel] = consistency_level # figure out if consistency level for this request is session is_session_consistency = (consistency_level == documents.ConsistencyLevel.Session) # set session token if required if is_session_consistency is True: # if there is a token set via option, then use it to override default if options.get('sessionToken'): headers[http_constants.HttpHeaders.SessionToken] = options['sessionToken'] else: # check if the client's default consistency is session (and request consistency level is same), # then update from session container if default_client_consistency_level == documents.ConsistencyLevel.Session: # populate session token from the client's session container headers[http_constants.HttpHeaders.SessionToken] = ( document_client.session.get_session_token(path)) if options.get('enableScanInQuery'): headers[http_constants.HttpHeaders.EnableScanInQuery] = ( options['enableScanInQuery']) if options.get('resourceTokenExpirySeconds'): headers[http_constants.HttpHeaders.ResourceTokenExpiry] = ( options['resourceTokenExpirySeconds']) if options.get('offerType'): headers[http_constants.HttpHeaders.OfferType] = options['offerType'] if options.get('offerThroughput'): headers[http_constants.HttpHeaders.OfferThroughput] = options['offerThroughput'] if 'partitionKey' in options: # if partitionKey value is Undefined, serialize it as {} to be consistent with other SDKs if options.get('partitionKey') is documents.Undefined: headers[http_constants.HttpHeaders.PartitionKey] = [{}] # else serialize using json dumps method which apart from regular values will serialize None into null else: headers[http_constants.HttpHeaders.PartitionKey] = json.dumps([options['partitionKey']]) if options.get('enableCrossPartitionQuery'): headers[http_constants.HttpHeaders.EnableCrossPartitionQuery] = options['enableCrossPartitionQuery'] if document_client.master_key: headers[http_constants.HttpHeaders.XDate] = ( datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')) if document_client.master_key or document_client.resource_tokens: authorization = auth.GetAuthorizationHeader(document_client, verb, path, resource_id, resource_type, headers) # urllib.quote throws when the input parameter is None if authorization: # -_.!~*'() are valid characters in url, and shouldn't be quoted. authorization = urllib_quote(authorization, '-_.!~*\'()') headers[http_constants.HttpHeaders.Authorization] = authorization if verb == 'post' or verb == 'put': if not headers.get(http_constants.HttpHeaders.ContentType): headers[http_constants.HttpHeaders.ContentType] = runtime_constants.MediaTypes.Json if not headers.get(http_constants.HttpHeaders.Accept): headers[http_constants.HttpHeaders.Accept] = runtime_constants.MediaTypes.Json if partition_key_range_id is not None: headers[http_constants.HttpHeaders.PartitionKeyRangeID] = partition_key_range_id if options.get('enableScriptLogging'): headers[http_constants.HttpHeaders.EnableScriptLogging] = options['enableScriptLogging'] return headers
def quote(text): # quote url path segments, but leave + and @ intact return urllib_quote(text, '/+@')
def GetHeaders(cosmos_client, default_headers, verb, path, resource_id, resource_type, options, partition_key_range_id = None): """Gets HTTP request headers. :param cosmos_client.CosmosClient cosmos_client: :param dict default_headers: :param str verb: :param str path: :param str resource_id: :param str resource_type: :param dict options: :param str partition_key_range_id: :return: The HTTP request headers. :rtype: dict """ headers = dict(default_headers) options = options or {} if cosmos_client._useMultipleWriteLocations: headers[http_constants.HttpHeaders.AllowTentativeWrites] = "true" pre_trigger_include = options.get('preTriggerInclude') if pre_trigger_include: headers[http_constants.HttpHeaders.PreTriggerInclude] = ( pre_trigger_include if isinstance(pre_trigger_include, str) else (',').join(pre_trigger_include)) post_trigger_include = options.get('postTriggerInclude') if post_trigger_include: headers[http_constants.HttpHeaders.PostTriggerInclude] = ( post_trigger_include if isinstance(post_trigger_include, str) else (',').join(post_trigger_include)) if options.get('maxItemCount'): headers[http_constants.HttpHeaders.PageSize] = options['maxItemCount'] access_condition = options.get('accessCondition') if access_condition: if access_condition['type'] == 'IfMatch': headers[http_constants.HttpHeaders.IfMatch] = access_condition['condition'] else: headers[http_constants.HttpHeaders.IfNoneMatch] = access_condition['condition'] if options.get('indexingDirective'): headers[http_constants.HttpHeaders.IndexingDirective] = ( options['indexingDirective']) consistency_level = None ''' get default client consistency level''' default_client_consistency_level = headers.get(http_constants.HttpHeaders.ConsistencyLevel) ''' set consistency level. check if set via options, this will override the default ''' if options.get('consistencyLevel'): consistency_level = options['consistencyLevel'] headers[http_constants.HttpHeaders.ConsistencyLevel] = consistency_level elif default_client_consistency_level is not None: consistency_level = default_client_consistency_level headers[http_constants.HttpHeaders.ConsistencyLevel] = consistency_level # figure out if consistency level for this request is session is_session_consistency = (consistency_level == documents.ConsistencyLevel.Session) # set session token if required if is_session_consistency is True and not IsMasterResource(resource_type): # if there is a token set via option, then use it to override default if options.get('sessionToken'): headers[http_constants.HttpHeaders.SessionToken] = options['sessionToken'] else: # check if the client's default consistency is session (and request consistency level is same), # then update from session container if default_client_consistency_level == documents.ConsistencyLevel.Session: # populate session token from the client's session container headers[http_constants.HttpHeaders.SessionToken] = ( cosmos_client.session.get_session_token(path)) if options.get('enableScanInQuery'): headers[http_constants.HttpHeaders.EnableScanInQuery] = ( options['enableScanInQuery']) if options.get('resourceTokenExpirySeconds'): headers[http_constants.HttpHeaders.ResourceTokenExpiry] = ( options['resourceTokenExpirySeconds']) if options.get('offerType'): headers[http_constants.HttpHeaders.OfferType] = options['offerType'] if options.get('offerThroughput'): headers[http_constants.HttpHeaders.OfferThroughput] = options['offerThroughput'] if 'partitionKey' in options: # if partitionKey value is Undefined, serialize it as {} to be consistent with other SDKs if options.get('partitionKey') is documents.Undefined: headers[http_constants.HttpHeaders.PartitionKey] = [{}] # else serialize using json dumps method which apart from regular values will serialize None into null else: headers[http_constants.HttpHeaders.PartitionKey] = json.dumps([options['partitionKey']]) if options.get('enableCrossPartitionQuery'): headers[http_constants.HttpHeaders.EnableCrossPartitionQuery] = options['enableCrossPartitionQuery'] if options.get('populateQueryMetrics'): headers[http_constants.HttpHeaders.PopulateQueryMetrics] = options['populateQueryMetrics'] if cosmos_client.master_key: headers[http_constants.HttpHeaders.XDate] = ( datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')) if cosmos_client.master_key or cosmos_client.resource_tokens: authorization = auth.GetAuthorizationHeader(cosmos_client, verb, path, resource_id, IsNameBased(resource_id), resource_type, headers) # urllib.quote throws when the input parameter is None if authorization: # -_.!~*'() are valid characters in url, and shouldn't be quoted. authorization = urllib_quote(authorization, '-_.!~*\'()') headers[http_constants.HttpHeaders.Authorization] = authorization if verb == 'post' or verb == 'put': if not headers.get(http_constants.HttpHeaders.ContentType): headers[http_constants.HttpHeaders.ContentType] = runtime_constants.MediaTypes.Json if not headers.get(http_constants.HttpHeaders.Accept): headers[http_constants.HttpHeaders.Accept] = runtime_constants.MediaTypes.Json if partition_key_range_id is not None: headers[http_constants.HttpHeaders.PartitionKeyRangeID] = partition_key_range_id if options.get('enableScriptLogging'): headers[http_constants.HttpHeaders.EnableScriptLogging] = options['enableScriptLogging'] if options.get('offerEnableRUPerMinuteThroughput'): headers[http_constants.HttpHeaders.OfferIsRUPerMinuteThroughputEnabled] = options['offerEnableRUPerMinuteThroughput'] if options.get('disableRUPerMinuteUsage'): headers[http_constants.HttpHeaders.DisableRUPerMinuteUsage] = options['disableRUPerMinuteUsage'] if options.get('changeFeed') is True: # On REST level, change feed is using IfNoneMatch/ETag instead of continuation. if_none_match_value = None if options.get('continuation'): if_none_match_value = options['continuation'] elif options.get('isStartFromBeginning') and options['isStartFromBeginning'] == False: if_none_match_value = '*' if if_none_match_value: headers[http_constants.HttpHeaders.IfNoneMatch] = if_none_match_value headers[http_constants.HttpHeaders.AIM] = http_constants.HttpHeaders.IncrementalFeedHeaderValue else: if options.get('continuation'): headers[http_constants.HttpHeaders.Continuation] = (options['continuation']) if options.get('populatePartitionKeyRangeStatistics'): headers[http_constants.HttpHeaders.PopulatePartitionKeyRangeStatistics] = options['populatePartitionKeyRangeStatistics'] if options.get('populateQuotaInfo'): headers[http_constants.HttpHeaders.PopulateQuotaInfo] = options['populateQuotaInfo'] return headers
def GetHeaders(document_client, default_headers, verb, path, resource_id, resource_type, options, partition_key_range_id=None): """Gets HTTP request headers. :Parameters: - `document_client`: document_client.DocumentClient - `default_headers`: dict - `verb`: str - `path`: str - `resource_id`: str - `resource_type`: str - `options`: dict - `partition_key_range_id` : str :Returns: dict, the HTTP request headers. """ headers = dict(default_headers) options = options or {} if options.get('continuation'): headers[http_constants.HttpHeaders.Continuation] = ( options['continuation']) pre_trigger_include = options.get('preTriggerInclude') if pre_trigger_include: headers[http_constants.HttpHeaders.PreTriggerInclude] = ( pre_trigger_include if isinstance(pre_trigger_include, str) else (',').join(pre_trigger_include)) post_trigger_include = options.get('postTriggerInclude') if post_trigger_include: headers[http_constants.HttpHeaders.PostTriggerInclude] = ( post_trigger_include if isinstance(post_trigger_include, str) else (',').join(post_trigger_include)) if options.get('maxItemCount'): headers[http_constants.HttpHeaders.PageSize] = options['maxItemCount'] access_condition = options.get('accessCondition') if access_condition: if access_condition['type'] == 'IfMatch': headers[http_constants.HttpHeaders. IfMatch] = access_condition['condition'] else: headers[http_constants.HttpHeaders. IfNoneMatch] = access_condition['condition'] if options.get('indexingDirective'): headers[http_constants.HttpHeaders.IndexingDirective] = ( options['indexingDirective']) consistency_level = None ''' get default client consistency level''' default_client_consistency_level = headers.get( http_constants.HttpHeaders.ConsistencyLevel) ''' set consistency level. check if set via options, this will override the default ''' if options.get('consistencyLevel'): consistency_level = options['consistencyLevel'] headers[ http_constants.HttpHeaders.ConsistencyLevel] = consistency_level elif default_client_consistency_level is not None: consistency_level = default_client_consistency_level headers[ http_constants.HttpHeaders.ConsistencyLevel] = consistency_level # figure out if consistency level for this request is session is_session_consistency = ( consistency_level == documents.ConsistencyLevel.Session) # set session token if required if is_session_consistency is True: # if there is a token set via option, then use it to override default if options.get('sessionToken'): headers[http_constants.HttpHeaders. SessionToken] = options['sessionToken'] else: # check if the client's default consistency is session (and request consistency level is same), # then update from session container if default_client_consistency_level == documents.ConsistencyLevel.Session: # populate session token from the client's session container headers[http_constants.HttpHeaders.SessionToken] = ( document_client.session.get_session_token(path)) if options.get('enableScanInQuery'): headers[http_constants.HttpHeaders.EnableScanInQuery] = ( options['enableScanInQuery']) if options.get('resourceTokenExpirySeconds'): headers[http_constants.HttpHeaders.ResourceTokenExpiry] = ( options['resourceTokenExpirySeconds']) if options.get('offerType'): headers[http_constants.HttpHeaders.OfferType] = options['offerType'] if options.get('offerThroughput'): headers[http_constants.HttpHeaders. OfferThroughput] = options['offerThroughput'] if 'partitionKey' in options: # if partitionKey value is Undefined, serialize it as {} to be consistent with other SDKs if options.get('partitionKey') is documents.Undefined: headers[http_constants.HttpHeaders.PartitionKey] = [{}] # else serialize using json dumps method which apart from regular values will serialize None into null else: headers[http_constants.HttpHeaders.PartitionKey] = json.dumps( [options['partitionKey']]) if options.get('enableCrossPartitionQuery'): headers[ http_constants.HttpHeaders. EnableCrossPartitionQuery] = options['enableCrossPartitionQuery'] if document_client.master_key: headers[http_constants.HttpHeaders.XDate] = ( datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S GMT')) if document_client.master_key or document_client.resource_tokens: authorization = auth.GetAuthorizationHeader(document_client, verb, path, resource_id, resource_type, headers) # urllib.quote throws when the input parameter is None if authorization: # -_.!~*'() are valid characters in url, and shouldn't be quoted. authorization = urllib_quote(authorization, '-_.!~*\'()') headers[http_constants.HttpHeaders.Authorization] = authorization if verb == 'post' or verb == 'put': if not headers.get(http_constants.HttpHeaders.ContentType): headers[http_constants.HttpHeaders. ContentType] = runtime_constants.MediaTypes.Json if not headers.get(http_constants.HttpHeaders.Accept): headers[http_constants.HttpHeaders. Accept] = runtime_constants.MediaTypes.Json if partition_key_range_id is not None: headers[http_constants.HttpHeaders. PartitionKeyRangeID] = partition_key_range_id if options.get('enableScriptLogging'): headers[http_constants.HttpHeaders. EnableScriptLogging] = options['enableScriptLogging'] if options.get('offerEnableRUPerMinuteThroughput'): headers[http_constants.HttpHeaders. OfferIsRUPerMinuteThroughputEnabled] = options[ 'offerEnableRUPerMinuteThroughput'] if options.get('disableRUPerMinuteUsage'): headers[http_constants.HttpHeaders. DisableRUPerMinuteUsage] = options['disableRUPerMinuteUsage'] return headers