def test_auth_failed(username, password, with_ssl, ssl_params):
ssl_params['use_ssl'] = with_ssl
with pytest.raises(AuthenticationError):
client = Client(username=username, password=password, **ssl_params)
with client.connect("127.0.0.1", 10801):
pass
def test_auth_success(with_ssl, ssl_params):
ssl_params['use_ssl'] = with_ssl
client = Client(username=DEFAULT_IGNITE_USERNAME,
password=DEFAULT_IGNITE_PASSWORD,
**ssl_params)
with client.connect("127.0.0.1", 10801):
assert all(node.alive for node in client._nodes)
def inner():
client = Client(**kwargs)
with client.connect("127.0.0.1", 10801):
with get_or_create_cache(client, 'test-cache') as cache:
cache.put(1, 1)
assert cache.get(1) == 1
def __init__(self, url='127.0.0.1', port=10800):
if ("IGNITE_CLUSTER_IP" in os.environ):
url = os.environ["IGNITE_CLUSTER_IP"]
if ("IGNITE_CLUSTER_PORT" in os.environ):
port = int(os.environ["IGNITE_CLUSTER_PORT"])
print(url, port)
self.client = Client()
self.client.connect(url, port)
def get_data(self) -> list:
client = Client()
client.connect('localhost', 10800)
streams_cache = client.get_or_create_cache(
json.loads(self.__stream_name)['Stream'])
stream = streams_cache.scan()
return list(stream)
def skip_if_no_affinity(request, server1):
client = Client(partition_aware=True)
client.connect('127.0.0.1', 10801)
if not client.partition_awareness_supported_by_protocol:
pytest.skip(
f'skipped {request.node.name}, partition awareness is not supported.'
)
def main():
client = Client()
client.connect('127.0.0.1', 10800)
my_cache = client.get_or_create_cache('my cache')
my_cache.put('my key', 42)
result = my_cache.get('my key')
print('my key: {}'.format(result))
class IgniteStorage(LightStorage):
"""
Apache Ignite storage for Light Requests and Responses.
:param host: Ignite service hostname or IP address.
:param port: Ignite service port.
:param request_cache_name: The cache where LightRequests are stored.
:param response_cache_name: The cache where LightResponses are stored.
:param timeout: Timeout (in seconds) for socket operations.
"""
def __init__(self,
host: str,
port: int,
request_cache_name: str,
response_cache_name: str,
timeout: int = 30):
self.host = host
self.port = port
self.request_cache_name = request_cache_name
self.response_cache_name = response_cache_name
self.timeout = timeout
self._client = None # type: Optional[Client]
def get_cache(self, cache_name: str) -> Cache:
"""Get an Ignite Cache."""
if self._client is None:
self._client = Client(timeout=self.timeout)
self._client.connect(self.host, self.port)
return self._client.get_cache(cache_name)
def pop_light_request(self, uid: str) -> Optional[LightRequest]:
"""Look up a LightRequest by a unique id and then remove it."""
data = self.get_cache(self.request_cache_name).get_and_remove(uid)
LOGGER.debug('Got Light Request from cache: id=%r, data=%s', uid, data)
return LightRequest().load_xml(
parse_xml(data)) if data is not None else None
def pop_light_response(self, uid: str) -> Optional[LightResponse]:
"""Look up a LightResponse by a unique id and then remove it."""
data = self.get_cache(self.response_cache_name).get_and_remove(uid)
LOGGER.debug('Got Light Response from cache: id=%r, data=%s', uid,
data)
return LightResponse().load_xml(
parse_xml(data)) if data is not None else None
def put_light_request(self, uid: str, request: LightRequest) -> None:
"""Store a LightRequest under a unique id."""
data = dump_xml(request.export_xml()).decode('utf-8')
LOGGER.debug('Store Light Request to cache: id=%r, data=%s', uid, data)
self.get_cache(self.request_cache_name).put(uid, data)
def put_light_response(self, uid: str, response: LightResponse) -> None:
"""Store a LightResponse under a unique id."""
data = dump_xml(response.export_xml()).decode('utf-8')
LOGGER.debug('Store Light Response to cache: id=%r, data=%s', uid,
data)
self.get_cache(self.response_cache_name).put(uid, data)
def main():
client = Client()
client.connect('127.0.0.1', 10800)
query = ''
result = client.sql(query)
for row in result:
print(row)
class Database:
def __init__(self, url='127.0.0.1', port=10800):
self.client = Client()
self.client.connect(url, port)
def getClient(self):
return self.client
def closeClient(self, client):
client.close()
def expiry_policy_supported(request, server1):
client = Client()
with client.connect('127.0.0.1', 10801):
result = client.protocol_context.is_expiry_policy_supported()
if not result and request.node.get_closest_marker(
'skip_if_no_expiry_policy'):
pytest.skip(
f'skipped {request.node.name}, ExpiryPolicy APIis not supported.'
)
return result
class AuxiliaryIgniteStorage(AuxiliaryStorage):
"""
Apache Ignite storage for auxiliary data.
:param host: Ignite service hostname or IP address.
:param port: Ignite service port.
:param cache_name: The cache where data are stored.
:param timeout: Timeout (in seconds) for socket operations.
"""
def __init__(self,
host: str,
port: int,
cache_name: str,
timeout: int = 30,
prefix: str = None):
self.host = host
self.port = port
self.cache_name = cache_name
self.timeout = timeout
self.prefix = prefix
self._client = None # type: Optional[Client]
def get_cache(self, cache_name: str) -> Cache:
"""Get an Ignite Cache."""
if self._client is None:
self._client = Client(timeout=self.timeout)
self._client.connect(self.host, self.port)
return self._client.get_cache(cache_name)
def pop(self, uid: str) -> Optional[Dict[str, Any]]:
"""Look up data by a unique id and then remove it."""
if self.prefix:
uid = self.prefix + uid
data = self.get_cache(self.cache_name).get_and_remove(uid)
LOGGER.debug('Got data from cache: id=%r, data=%s', uid, data)
return json.loads(data) if data is not None else None
def put(self, uid: str, data: Dict[str, Any]) -> None:
"""
Store data under a unique id.
Data must be JSON-serializable.
"""
if self.prefix:
uid = self.prefix + uid
LOGGER.debug('Store data to cache: id=%r, data=%s', uid, data)
self.get_cache(self.cache_name).put(uid,
json.dumps(data, sort_keys=True))
class Database:
def __init__(self, url='127.0.0.1', port=10800):
if ("IGNITE_CLUSTER_IP" in os.environ):
url = os.environ["IGNITE_CLUSTER_IP"]
if ("IGNITE_CLUSTER_PORT" in os.environ):
port = int(os.environ["IGNITE_CLUSTER_PORT"])
print(url, port)
self.client = Client()
self.client.connect(url, port)
def getClient(self):
return self.client
def closeClient(self, client):
client.close()
def artistMaster(request):
loginUser = request.user
nodes = [
('127.0.0.1', 10800),
('127.0.0.1', 10801),
]
client = Client()
client.connect(nodes)
#QUERY = ''' SELECT ID, TRACKID, TRACKDATA FROM PUBLIC.SONG_DATA ORDER BY ID DESC; '''
QUERY = ''' SELECT TRACKNAME, ALBUMNAME, ARTISTNAME, GENRE FROM PUBLIC.SONG_DATA ORDER BY ID DESC; '''
#print(QUERY)
result = client.sql(
QUERY,
)
datatohtml = []
'''for row in result:
try:
dataparsed = json.loads(row[2])
try:
genre = dataparsed.get("primary_genres").get("music_genre_list")[0].get("music_genre").get("music_genre_name")
except Exception as e:
genre = "-"
dataauto ={
"track_name":dataparsed.get("track_name"),
"artist_name":dataparsed.get("artist_name"),
"album_name":dataparsed.get("album_name"),
"genre":genre
}
datatohtml.append(dataauto)
except Exception as e:
pass
# print(datatohtml)'''
for row in result:
dataauto = {
"track_name":row[0],
"artist_name":row[2],
"album_name":row[1],
"genre":row[3],
}
datatohtml.append(dataauto)
return render(request,"ArtistMaster.html",{"loginUser":loginUser,"datatohtml":datatohtml})
def test_client_with_failed_server(request, with_partition_awareness):
srv = start_ignite(idx=4)
try:
client = Client(partition_aware=with_partition_awareness)
with client.connect([("127.0.0.1", 10804)]):
cache = client.get_or_create_cache(request.node.name)
cache.put(1, 1)
kill_process_tree(srv.pid)
if with_partition_awareness:
ex_class = (ReconnectError, ConnectionResetError)
else:
ex_class = ConnectionResetError
with pytest.raises(ex_class):
cache.get(1)
finally:
kill_process_tree(srv.pid)
def main():
print("Running sync ExpiryPolicy example.")
client = Client()
with client.connect('127.0.0.1', 10800):
print("Create cache with expiry policy.")
try:
ttl_cache = client.create_cache({
PROP_NAME:
'test',
PROP_EXPIRY_POLICY:
ExpiryPolicy(create=1.0)
})
except NotSupportedByClusterError:
print(
"'ExpiryPolicy' API is not supported by cluster. Finishing...")
return
try:
ttl_cache.put(1, 1)
time.sleep(0.5)
print(f"key = {1}, value = {ttl_cache.get(1)}")
# key = 1, value = 1
time.sleep(1.2)
print(f"key = {1}, value = {ttl_cache.get(1)}")
# key = 1, value = None
finally:
ttl_cache.destroy()
print("Create simple Cache and set TTL through `with_expire_policy`")
simple_cache = client.create_cache('test')
try:
ttl_cache = simple_cache.with_expire_policy(access=1.0)
ttl_cache.put(1, 1)
time.sleep(0.5)
print(f"key = {1}, value = {ttl_cache.get(1)}")
# key = 1, value = 1
time.sleep(1.7)
print(f"key = {1}, value = {ttl_cache.get(1)}")
# key = 1, value = None
finally:
simple_cache.destroy()
def client(connection_param):
client = Client(partition_aware=True, timeout=CLIENT_SOCKET_TIMEOUT)
try:
client.connect(connection_param)
yield client
finally:
client.close()
def client():
client = Client()
try:
client.connect('127.0.0.1', 10801)
yield client
finally:
client.close()
def client_routed():
client = Client(partition_aware=True)
try:
client.connect(client_routed_connection_string)
yield client
finally:
client.close()
def client(
ignite_host,
ignite_port,
timeout,
use_ssl,
ssl_keyfile,
ssl_certfile,
ssl_ca_certfile,
ssl_cert_reqs,
ssl_ciphers,
ssl_version,
username,
password,
):
client = Client(
timeout=timeout,
use_ssl=use_ssl,
ssl_keyfile=ssl_keyfile,
ssl_certfile=ssl_certfile,
ssl_ca_certfile=ssl_ca_certfile,
ssl_cert_reqs=ssl_cert_reqs,
ssl_ciphers=ssl_ciphers,
ssl_version=ssl_version,
username=username,
password=password,
)
client.connect(ignite_host, ignite_port)
yield client
for cache_name in cache_get_names(client).value:
cache_destroy(client, cache_name)
client.close()
def export_users_csv(request):
print("Hello")
nodes = [
('127.0.0.1', 10800),
('127.0.0.1', 10801),
]
client = Client()
client.connect(nodes)
yesterday = date.today() - timedelta(days=1)
datecsv = yesterday.strftime('%m/%d/%Y')
print(datecsv)
QUERY = ''' SELECT ID, APPID, EVENTS, DEVICE_ID, "TIMESTAMP", MTIMESTAMP, UPTIMESTAMP, IPADDRESS, CITY, COUNTRY, EVENTNAME, CREATED_DATE, CREATED_TIME FROM PUBLIC.EVENTSDATA WHERE APPID != '5def994ce96b09565e1f1ddd' AND CREATED_DATE = {} AND EVENTNAME!='_app_crash' AND ID!=855263 ORDER BY ID; '''.format("'"+datecsv+"'")
print(QUERY)
result = client.sql(
QUERY,
include_field_names=True,
)
print(next(result))
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="{}.csv"'.format(datecsv+"_IgniteEvents")
writer = csv.writer(response)
writer.writerow(['appid','deviceid','devicemodel','platform','apppackage','keyname','mobileoperator','app','song','album','pstate','source','ipaddress','city','station','duration','timestamp','to_char','created_time'])
i = 0
for row in result:
if i == 0:
print(row)
x = row[2]
y = json.loads(x)
if y.get("segmentation") is None:
writer.writerow([row[1],row[3],y.get("d"),y.get("p"),y.get("ap"),row[10],y.get("network").get("ope"),"","","","","",row[7],row[8],"","",row[4],row[11],row[12]])
else:
writer.writerow([row[1],row[3],y.get("d"),y.get("p"),y.get("ap"),row[10],y.get("network").get("ope"),y.get("segmentation").get("App"),y.get("segmentation").get("Song"),y.get("segmentation").get("Album"),y.get("segmentation").get("PState"),y.get("segmentation").get("Source"),row[7],row[8],y.get("segmentation").get("Station"),y.get("segmentation").get("Duration"),row[4],row[11],row[12]])
i = 1
else:
print(row)
x = row[5]
try:
y = json.loads(x)
except:
pass
if y.get("segmentation") is None:
writer.writerow([row[1],row[6],y.get("d"),y.get("p"),y.get("ap"),row[2],y.get("network").get("ope"),"","","","","",row[10],row[11],"","",row[7],row[3],row[4]])
else:
writer.writerow([row[1],row[6],y.get("d"),y.get("p"),y.get("ap"),row[2],y.get("network").get("ope"),y.get("segmentation").get("App"),y.get("segmentation").get("Song"),y.get("segmentation").get("Album"),y.get("segmentation").get("PState"),y.get("segmentation").get("Source"),row[10],row[11],y.get("segmentation").get("Station"),y.get("segmentation").get("Duration"),row[7],row[3],row[4]])
client.close()
return response
def client(
ignite_host, ignite_port, timeout, use_ssl, ssl_keyfile, ssl_certfile,
ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version,
username, password,
):
client = Client(
timeout=timeout,
use_ssl=use_ssl,
ssl_keyfile=ssl_keyfile,
ssl_certfile=ssl_certfile,
ssl_ca_certfile=ssl_ca_certfile,
ssl_cert_reqs=ssl_cert_reqs,
ssl_ciphers=ssl_ciphers,
ssl_version=ssl_version,
username=username,
password=password,
)
client.connect(ignite_host, ignite_port)
yield client
for cache_name in cache_get_names(client).value:
cache_destroy(client, cache_name)
client.close()
def test_cluster_set_active(with_persistence):
key = 42
val = 42
start_state = ClusterState.INACTIVE if with_persistence else ClusterState.ACTIVE
client = Client()
with client.connect([("127.0.0.1", 10801), ("127.0.0.1", 10802)]):
cluster = client.get_cluster()
assert cluster.get_state() == start_state
cluster.set_state(ClusterState.ACTIVE)
assert cluster.get_state() == ClusterState.ACTIVE
cache = client.get_or_create_cache("test_cache")
cache.put(key, val)
assert cache.get(key) == val
cluster.set_state(ClusterState.ACTIVE_READ_ONLY)
assert cluster.get_state() == ClusterState.ACTIVE_READ_ONLY
assert cache.get(key) == val
with pytest.raises(CacheError):
cache.put(key, val + 1)
cluster.set_state(ClusterState.INACTIVE)
assert cluster.get_state() == ClusterState.INACTIVE
with pytest.raises(CacheError):
cache.get(key)
with pytest.raises(CacheError):
cache.put(key, val + 1)
cluster.set_state(ClusterState.ACTIVE)
assert cluster.get_state() == ClusterState.ACTIVE
cache.put(key, val + 2)
assert cache.get(key) == val + 2
def test_client_with_recovered_server(request, with_partition_awareness):
srv = start_ignite(idx=4)
try:
client = Client(partition_aware=with_partition_awareness,
timeout=CLIENT_SOCKET_TIMEOUT)
with client.connect([("127.0.0.1", 10804)]):
cache = client.get_or_create_cache(request.node.name)
cache.put(1, 1)
# Kill and restart server
kill_process_tree(srv.pid)
srv = start_ignite(idx=4)
# First request may fail.
try:
cache.put(1, 2)
except connection_errors:
pass
# Retry succeeds
cache.put(1, 2)
assert cache.get(1) == 2
finally:
kill_process_tree(srv.pid)
def test_connection_context(connection_param, partition_aware):
is_partition_aware = partition_aware == 'with_partition_aware'
client = Client(partition_aware=is_partition_aware)
# Check context manager
with client.connect(connection_param):
__check_open(client, is_partition_aware)
__check_closed(client)
# Check standard way
try:
client.connect(connection_param)
__check_open(client, is_partition_aware)
finally:
client.close()
__check_closed(client)
def test_connection_error_with_incorrect_config(invalid_ssl_params):
with pytest.raises(ReconnectError):
client = Client(**invalid_ssl_params)
with client.connect([("127.0.0.1", 10801)]):
pass
from pyignite import Client
import ssl
#tag::no-ssl[]
client = Client(username='******', password='******', use_ssl=False)
#end::no-ssl[]
client = Client(
ssl_cert_reqs=ssl.CERT_REQUIRED,
ssl_keyfile='/path/to/key/file',
ssl_certfile='/path/to/client/cert',
ssl_ca_certfile='/path/to/trusted/cert/or/chain',
username='******',
password='******',
)
client.connect('localhost', 10800)
def test_handshake(
monkeypatch,
ignite_host,
ignite_port,
use_ssl,
ssl_keyfile,
ssl_certfile,
ssl_ca_certfile,
ssl_cert_reqs,
ssl_ciphers,
ssl_version,
username,
password,
):
client = Client(
use_ssl=use_ssl,
ssl_keyfile=ssl_keyfile,
ssl_certfile=ssl_certfile,
ssl_ca_certfile=ssl_ca_certfile,
ssl_cert_reqs=ssl_cert_reqs,
ssl_ciphers=ssl_ciphers,
ssl_version=ssl_version,
username=username,
password=password,
)
client._socket = client._wrap(
socket.socket(socket.AF_INET, socket.SOCK_STREAM))
client.socket.connect((ignite_host, ignite_port))
hs_request = HandshakeRequest(username, password)
client.send(hs_request)
hs_response = read_response(client)
assert hs_response['op_code'] != 0
client.close()
# intentionally pass wrong protocol version
from pyignite.connection import handshake
monkeypatch.setattr(handshake, 'PROTOCOL_VERSION_MAJOR', 10)
client._socket = client._wrap(
socket.socket(socket.AF_INET, socket.SOCK_STREAM))
client.socket.connect((ignite_host, ignite_port))
hs_request = HandshakeRequest(username, password)
client.send(hs_request)
hs_response = read_response(client)
assert hs_response['op_code'] == 0
client.close()
#
# https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from collections import OrderedDict
from pyignite import Client, GenericObjectMeta
from pyignite.datatypes import DoubleObject, IntObject, String
from pyignite.datatypes.prop_codes import *
client = Client()
client.connect('127.0.0.1', 10800)
student_cache = client.create_cache({
PROP_NAME: 'SQL_PUBLIC_STUDENT',
PROP_SQL_SCHEMA: 'PUBLIC',
PROP_QUERY_ENTITIES: [
{
'table_name': 'Student'.upper(),
'key_field_name': 'SID',
'key_type_name': 'java.lang.Integer',
'field_name_aliases': [],
'query_fields': [
{
'name': 'SID',
'type_name': 'java.lang.Integer',
from collections import OrderedDict
from pyignite import Client, GenericObjectMeta
from pyignite.datatypes import *
class Person(metaclass=GenericObjectMeta, schema=OrderedDict([
('first_name', String),
('last_name', String),
('age', IntObject),
])):
pass
client = Client()
client.connect('localhost', 10800)
person_cache = client.get_or_create_cache('person')
person_cache.put(
1, Person(first_name='Ivan', last_name='Ivanov', age=33)
)
person = person_cache.get(1)
print(person.__class__.__name__)
# Person
print(person.__class__ is Person)
# True if `Person` was registered automatically (on writing)
# or manually (using `client.register_binary_type()` method).
from pyignite import Client
client = Client()
client.connect('127.0.0.1', 10800)
PRODUCT_CREATE_TABLE = '''
create table product(
product_id VARCHAR PRIMARY KEY,
product_url VARCHAR
)
'''
client.sql(PRODUCT_CREATE_TABLE)
SIMILARPRODUCTS_CREATE_TABLE = '''
create table similarproducts(
product_id VARCHAR ,
similar_product_id VARCHAR ,
similariy_score DOUBLE ,
PRIMARY KEY (product_id, similar_product_id)
) WITH "affinityKey=similar_product_id"
'''
client.sql(SIMILARPRODUCTS_CREATE_TABLE)
PRODUCT_INSERT_QUERY = '''
insert into product values (? , ?)
'''
SIMILARPRODUCTS_INSERT_QUERY = '''
insert into similarproducts values (? , ?, ?)
'''
# Code for inserting dummy data
INSERT_PRODUCT_DATA= []
for i in range(10000) :
# See the License for the specific language governing permissions and
# limitations under the License.
from pyignite import Client
from pyignite.datatypes.cache_config import CacheMode
from pyignite.datatypes.prop_codes import *
from pyignite.exceptions import SocketError
nodes = [
('127.0.0.1', 10800),
('127.0.0.1', 10801),
('127.0.0.1', 10802),
]
client = Client(timeout=4.0)
client.connect(nodes)
print('Connected to {}'.format(client))
my_cache = client.get_or_create_cache({
PROP_NAME: 'my_cache',
PROP_CACHE_MODE: CacheMode.REPLICATED,
})
my_cache.put('test_key', 0)
# abstract main loop
while True:
try:
# do the work
test_value = my_cache.get('test_key')
my_cache.put('test_key', test_value + 1)
# include, and the License does not grant to you, the right to Sell the Software.
# For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you
# under the License to provide to third parties, for a fee or other consideration (including without
# limitation fees for hosting or consulting/ support services related to the Software), a product or
# service whose value derives, entirely or substantially, from the functionality of the Software.
# Any license notice or attribution required by the License must also include this Commons Clause
# License Condition notice.
#
# For purposes of the clause above, the “Licensor” is Copyright 2019 GridGain Systems, Inc.,
# the “License” is the Apache License, Version 2.0, and the Software is the GridGain Community
# Edition software provided with this notice.
from pyignite import Client
from pyignite.datatypes import CharObject, ShortObject
client = Client()
client.connect('127.0.0.1', 10800)
my_cache = client.get_or_create_cache('my cache')
my_cache.put('my key', 42)
# value ‘42’ takes 9 bytes of memory as a LongObject
my_cache.put('my key', 42, value_hint=ShortObject)
# value ‘42’ takes only 3 bytes as a ShortObject
my_cache.put('a', 1)
# ‘a’ is a key of type String
my_cache.put('a', 2, key_hint=CharObject)
# another key ‘a’ of type CharObject was created
# 'report_date': DateObject,
# 'purpose': String,
# 'sum': DecimalObject,
# 'recipient': String,
# 'cashier_id': LongObject,
# }
class ExpenseVoucher(
metaclass=GenericObjectMeta,
schema=old_schema,
):
pass
client = Client()
client.connect('127.0.0.1', 10800)
accounting = client.get_or_create_cache('accounting')
for key, value in old_data:
accounting.put(key, ExpenseVoucher(**value))
data_classes = client.query_binary_type('ExpenseVoucher')
print(data_classes)
# {
# -231598180: <class '__main__.ExpenseVoucher'>
# }
s_id, data_class = data_classes.popitem()
schema = data_class.schema
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pyignite import Client
from pyignite.datatypes import CharObject, ShortObject
client = Client()
client.connect('127.0.0.1', 10800)
my_cache = client.get_or_create_cache('my cache')
my_cache.put('my key', 42)
# value ‘42’ takes 9 bytes of memory as a LongObject
my_cache.put('my key', 42, value_hint=ShortObject)
# value ‘42’ takes only 3 bytes as a ShortObject
my_cache.put('a', 1)
# ‘a’ is a key of type String
my_cache.put('a', 2, key_hint=CharObject)
# another key ‘a’ of type CharObject was created
['CHN', 'Dong', False, Decimal('0.2')],
['CHN', 'Hui', False, Decimal('0.8')],
['CHN', 'Mantšu', False, Decimal('0.9')],
['CHN', 'Miao', False, Decimal('0.7')],
['CHN', 'Mongolian', False, Decimal('0.4')],
['CHN', 'Puyi', False, Decimal('0.2')],
['CHN', 'Tibetan', False, Decimal('0.4')],
['CHN', 'Tujia', False, Decimal('0.5')],
['CHN', 'Uighur', False, Decimal('0.6')],
['CHN', 'Yi', False, Decimal('0.6')],
['CHN', 'Zhuang', False, Decimal('1.4')],
]
# establish connection
client = Client()
client.connect('127.0.0.1', 10800)
# create tables
for query in [
COUNTRY_CREATE_TABLE_QUERY,
CITY_CREATE_TABLE_QUERY,
LANGUAGE_CREATE_TABLE_QUERY,
]:
client.sql(query)
# create indices
for query in [CITY_CREATE_INDEX, LANGUAGE_CREATE_INDEX]:
client.sql(query)
# load data
['CHN', 'Dong', False, Decimal('0.2')],
['CHN', 'Hui', False, Decimal('0.8')],
['CHN', 'Mantšu', False, Decimal('0.9')],
['CHN', 'Miao', False, Decimal('0.7')],
['CHN', 'Mongolian', False, Decimal('0.4')],
['CHN', 'Puyi', False, Decimal('0.2')],
['CHN', 'Tibetan', False, Decimal('0.4')],
['CHN', 'Tujia', False, Decimal('0.5')],
['CHN', 'Uighur', False, Decimal('0.6')],
['CHN', 'Yi', False, Decimal('0.6')],
['CHN', 'Zhuang', False, Decimal('1.4')],
]
# establish connection
client = Client()
client.connect('127.0.0.1', 10800)
# create tables
for query in [
COUNTRY_CREATE_TABLE_QUERY,
CITY_CREATE_TABLE_QUERY,
LANGUAGE_CREATE_TABLE_QUERY,
]:
client.sql(query)
# create indices
for query in [CITY_CREATE_INDEX, LANGUAGE_CREATE_INDEX]:
client.sql(query)
# load data
def export_users_csv():
print("Hello")
nodes = [
('35.154.247.92', 10800),
('35.154.247.92', 10801),
]
conn = psycopg2.connect(
dbname="VTIONData",
host="vtionproddb.chgz4nqwpdta.ap-south-1.rds.amazonaws.com",
user="******",
password="******")
cur = conn.cursor()
client = Client()
client.connect(nodes)
yesterday = date.today() - timedelta(days=1)
yesterdayMidnight = int(time.mktime(yesterday.timetuple())) - 19800
print("FROM : ", yesterdayMidnight)
todayMidnight = yesterdayMidnight + 86400
print(todayMidnight)
yesterdayMidnight = '1579199400'
todayMidnight = '1579285800'
QUERY = ''' SELECT ID, APPID, EVENTS, DEVICE_ID, "TIMESTAMP", MTIMESTAMP, UPTIMESTAMP, IPADDRESS, CITY,
COUNTRY, EVENTNAME, CREATED_DATE, CREATED_TIME FROM PUBLIC.EVENTSDATA WHERE
"TIMESTAMP" > '{}' AND "TIMESTAMP" < '{}' ;'''.format(
yesterdayMidnight, todayMidnight)
result = client.sql(
QUERY,
include_field_names=True,
)
next(result)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
print(BASE_DIR)
namespac = BASE_DIR + str("/target/Data_" +
yesterday.strftime('%m_%d_%Y') + '.csv')
with open(namespac, 'w+') as writeFile:
print("START : ", time.localtime(time.time()))
writer = csv.writer(writeFile)
writer.writerow([
'appid', 'DeviceId', 'vtionid', 'deviceModel', 'platform',
'apppackage', 'keyname', 'mobileOperator', 'app', 'song', 'album',
'Pstate', 'Program', 'Episode', 'Source', 'ipaddress', 'city',
'station', 'duration', 'timestamp', 'created_date', 'created_time',
'stationName', 'artist_name', 'genre', 'education', 'ownership',
'nccs_code', 'age', 'gender', 'number', 'Uninstall',
'Reward_option'
])
i = 0
for row in result:
nccs = age = gender = number = status = ''
artist_name = ''
genre = ''
try:
cur.execute(
'''SELECT nccs_code FROM public.installdata WHERE deviceid = '{}';'''
.format(row[6]))
nccs_result = cur.fetchone()
nccs_res = nccs_result[0]
except Exception as e:
nccs_res = ''
pass
if i == 0:
x = row[2]
i = 1
else:
x = row[5]
urlSafeEncodedBytes = base64.urlsafe_b64encode(
row[6].encode("utf-8"))
vtionid = str(urlSafeEncodedBytes, "utf-8")
try:
y = json.loads(x)
if y.get("segmentation") is None:
writer.writerow([
row[1], row[6], vtionid,
y.get("d"),
y.get("p"),
y.get("ap"), row[2],
y.get("network").get("ope"), "", "", "", "", "",
"", "", row[10], row[11], "", "", row[7], row[3],
row[4]
])
else:
if y.get("ap") == "com.video.meter":
song = y.get("segmentation").get("Song")
app = y.get("segmentation").get("App")
education = y.get("segmentation").get(
"Highest Education")
ownership = y.get("segmentation").get("Ownership")
if row[2] == 'FM_Tuned':
try:
QUERYa = ''' SELECT STATION FROM MASTERTABLEDEMO WHERE CITY = '{}'
AND FREQUENCY = '{}'; '''.format(
row[11],
y.get("segmentation").get("Station"))
resulta = client.sql(
QUERYa,
include_field_names=True,
)
next(resulta)
stationName = [rowa for rowa in resulta]
writer.writerow([
row[1], row[6], vtionid,
y.get("d"),
y.get("p"),
y.get("ap"), row[2],
y.get("network").get("ope"),
y.get("segmentation").get("App"),
y.get("segmentation").get("Song"),
y.get("segmentation").get("Album"),
y.get("segmentation").get("PState"),
"", "",
y.get("segmentation").get("Source"),
row[10], row[11],
y.get("segmentation").get("Station"),
y.get("segmentation").get("Duration"),
row[7], row[3], row[4],
stationName[0][0], "", "", "", "",
nccs_res
])
except Exception as e:
writer.writerow([
row[1], row[6], vtionid,
y.get("d"),
y.get("p"),
y.get("ap"), row[2],
y.get("network").get("ope"),
y.get("segmentation").get("App"),
y.get("segmentation").get("Song"),
y.get("segmentation").get("Album"),
y.get("segmentation").get("PState"),
"", "",
y.get("segmentation").get("Source"),
row[10], row[11],
y.get("segmentation").get("Station"),
y.get("segmentation").get("Duration"),
row[7], row[3], row[4], "", "", "", "",
"", nccs_res
])
elif row[2] == 'Video_Tuned' or row[
2] == 'Video_off':
try:
writer.writerow([
row[1], row[6], vtionid,
y.get("d"),
y.get("p"),
y.get("ap"), row[2],
y.get("network").get("ope"),
y.get("segmentation").get("App"),
y.get("segmentation").get("Song"),
y.get("segmentation").get("Album"),
y.get("segmentation").get("PState"),
y.get("segmentation").get("Program"),
y.get("segmentation").get("Episode"),
y.get("segmentation").get("Source"),
row[10], row[11], "",
y.get("segmentation").get("Duration"),
row[7], row[3], row[4], "", "", "", "",
"", nccs_res
])
except Exception as e:
pass
elif row[2] == 'Audio_Tuned':
try:
if song:
if song.isdigit():
continue
song = song.strip()
filter_new = [
'Download', 'download', 'AUD',
'Advertise', '%'
]
status_filter = (
filter_new[0] in song
or filter_new[1] in song
or filter_new[2] in song
or filter_new[3] in song
or filter_new[4] in song)
if str(status_filter) == 'False':
album = y.get("segmentation").get(
"Album")
if album is None:
album_filter = False
else:
a_filter = [
'unknown', 'Advertise',
'Sponsored'
]
album_filter = (
a_filter[0] in album
or a_filter[1] in album
or a_filter[2] in album)
if str(album_filter) == 'False':
QUERY = '''SELECT ARTISTNAME, GENRE FROM PUBLIC.SONG_DATA
WHERE TRACKNAME = '{}';'''.format(
song)
details = client.sql(
QUERY,
include_field_names=True,
)
next(details)
for det in details:
artist_name = det[0]
genre = det[1]
if artist_name:
pass
else:
QUERY = '''SELECT ARTISTNAME, GENRE FROM PUBLIC.SONG_DATA_ADD
WHERE TRACKNAME = '{}';'''.format(
song)
details = client.sql(
QUERY,
include_field_names=
True,
)
next(details)
for det in details:
artist_name = det[0]
genre = det[1]
if app == 'Amazon Music':
pass
if '%' in song or song.isdigit(
):
continue
else:
if 'ñ' in song:
song = song.replace(
'ñ', 'n')
else:
pass
song = deEmojify(
filter(
remove_bdata(song).
replace(
')',
'').replace(
'(', '')))
elif app == 'Spotify' or 'Spot' in app:
if '%' in song:
continue
else:
song = filter(
deEmojify(
remove_bdata(
song)))
elif app == 'Gaana':
if 'Gaana' in song or song.isdigit(
) or song == ',':
continue
else:
if 'ñ' in song or '├▒' in song:
song = song.replace(
'ñ',
'n').replace(
'├▒', 'n')
else:
pass
song = select_before_list(
remove_bdata(
deEmojify(song))
).lstrip('Ep 0123456789-.')
elif app == 'Google Play Music':
aa = song
if song.isdigit(
) or 'high quality' in song or song == ',':
continue
else:
song = domain(
remove_bdata(song)
).lstrip(
' 0123456789.-'
).strip()
if '128k' in song or '256k' in song or 'p3' in song:
song = song.replace(
'(128k)', ''
).replace(
'(256k)',
'').replace(
'128k', ''
).replace(
'256k', ''
).replace(
'.mp3', '')
song = select_before_list(
filter(
deEmojify(
song)))
elif app == 'Hungama':
song = filter(
remove_bdata(
deEmojify(song)))
elif app == 'i Music':
song = song.replace(
'-', ' ').replace(
':', ' ').replace(
' ',
' ').replace(
' ', ' '
).replace(
'.mp3', '')
if 'Unknown artist' in song or 'i Music' in song or '//' in song:
continue
song = deEmojify(
select_before_list(
filter(
domain(
remove_bdata(
song))
).lstrip(
' 0123456789.-'
).strip())).strip(
)
elif app == 'JioSaavn':
song = filter(
remove_bdata(song))
elif app == 'Saavn': #combine with jio savan
song = filter(
domain(
remove_bdata(
song)))
app = 'JioSaavn'
elif app == 'MX Player':
if '--:--:--' in song or 'Insufficient balance' in song or 'Expiry Date' in song:
continue
else:
song = deEmojify(
filter(
remove_bdata(
song)
).replace(
' ',
' ')).strip()
elif app == 'Music Player':
song = domain(
filter(
remove_bdata(song))
).replace(
' ', ' ').lstrip(
' 0123456789.-'
).strip()
elif app == 'Wynk Music':
song = remove_bdata(
deEmojify(song)
).lstrip(' 0123456789.-'
).strip()
if song.isdigit(
) or '%' in song:
continue
elif '(From' in song:
song = song.replace(
'(From', '')
else:
if '")' in song:
song = remove_bdata(
song.replace(
'\")', ''))
elif app == 'YouTube Music':
song = filter(
deEmojify(
remove_bdata(
song)))
else:
song = song
app = app
try:
if song:
if song.isdigit():
continue
song = song.strip()
writer.writerow([
row[1], row[6],
vtionid,
y.get("d"),
y.get("p"),
y.get("ap"),
row[2],
y.get("network"
).get("ope"),
app, song,
y.get(
"segmentation"
).get("Album"),
y.get(
"segmentation"
).get("PState"),
"", "",
y.get(
"segmentation"
).get("Source"),
row[10], row[11],
y.get(
"segmentation"
).get("Station"),
y.get(
"segmentation"
).get("Duration"),
row[7], row[3],
row[4], "",
artist_name, genre,
'', '', nccs_res,
'', '', '', ''
])
else:
pass
except Exception as e:
pass
else:
pass
else:
pass
except Exception as e:
pass
elif row[2] == 'Register' or row[2] == 'Profile':
if ownership:
try:
own = ownership.split(',')
if len(own) >= 9:
num_own = 9
else:
num_own = len(own)
cur.execute(
'''SELECT nccs_code FROM public.nccs_flat where education = '{}'
and ownership = '{}';'''.
format(education, num_own))
nccs = cur.fetchone()
except Exception as e:
pass
try:
try:
cur.execute(
'''SELECT payment_option FROM public.payment_option where deviceid = '{}';
'''.format(row[6]))
payment_option = cur.fetchone()
payment = payment_option[0]
except:
payment = 'Amazon voucher'
pass
age = y.get("segmentation").get("age")
if age:
pass
else:
age = y.get("segmentation").get("Age")
gender = y.get("segmentation").get(
"Gender")
if gender:
pass
else:
gender = y.get("segmentation").get(
"gender")
number = y.get("segmentation").get(
"Mobile Number")
num_status = "+" in number
if str(num_status) == 'True':
number = ''
else:
try:
cur.execute(
'''SELECT i_status FROM public.appsflyer where
number = '{}';'''.format(
number))
status_now = cur.fetchone()
status = status_now[0]
if status != 'True':
status = ''
except:
pass
try:
writer.writerow([
row[1], row[6], vtionid,
y.get("d"),
y.get("p"),
y.get("ap"), row[2],
y.get("network").get("ope"),
y.get("segmentation").get("App"),
y.get("segmentation").get("Song"),
y.get("segmentation").get("Album"),
y.get("segmentation").get(
"PState"), "", "",
y.get("segmentation").get(
"Source"), row[10], row[11],
y.get("segmentation").get(
"Station"),
y.get("segmentation").get(
"Duration"), row[7], row[3],
row[4], "", artist_name, genre,
education, ownership, nccs[0], age,
gender, number, status, payment
])
except Exception as e:
pass
writer.writerow([
row[1], row[6], vtionid,
y.get("d"),
y.get("p"),
y.get("ap"), row[2],
y.get("network").get("ope"),
y.get("segmentation").get("App"),
y.get("segmentation").get("Song"),
y.get("segmentation").get("Album"),
y.get("segmentation").get(
"PState"), "", "",
y.get("segmentation").get(
"Source"), row[10], row[11],
y.get("segmentation").get(
"Station"),
y.get("segmentation").get(
"Duration"), row[7], row[3],
row[4], "", artist_name, genre,
education, ownership, "", age,
gender, number, status, payment
])
except Exception as e:
pass
else:
writer.writerow([
row[1], row[6], vtionid,
y.get("d"),
y.get("p"),
y.get("ap"), row[2],
y.get("network").get("ope"),
y.get("segmentation").get("App"),
y.get("segmentation").get("Song"),
y.get("segmentation").get("Album"),
y.get("segmentation").get("PState"), "",
"",
y.get("segmentation").get("Source"),
row[10], row[11],
y.get("segmentation").get("Station"),
y.get("segmentation").get("Duration"),
row[7], row[3], row[4], "", artist_name,
genre, education, ownership, ""
])
except Exception as e:
pass
client.close()
writeFile.close()
def test_handshake(
monkeypatch,
ignite_host, ignite_port, use_ssl, ssl_keyfile, ssl_certfile,
ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version,
username, password,
):
client = Client(
use_ssl=use_ssl,
ssl_keyfile=ssl_keyfile,
ssl_certfile=ssl_certfile,
ssl_ca_certfile=ssl_ca_certfile,
ssl_cert_reqs=ssl_cert_reqs,
ssl_ciphers=ssl_ciphers,
ssl_version=ssl_version,
username=username,
password=password,
)
client._socket = client._wrap(
socket.socket(socket.AF_INET, socket.SOCK_STREAM)
)
client.socket.connect((ignite_host, ignite_port))
hs_request = HandshakeRequest(username, password)
client.send(hs_request)
hs_response = read_response(client)
assert hs_response['op_code'] != 0
client.close()
# intentionally pass wrong protocol version
from pyignite.connection import handshake
monkeypatch.setattr(handshake, 'PROTOCOL_VERSION_MAJOR', 10)
client._socket = client._wrap(
socket.socket(socket.AF_INET, socket.SOCK_STREAM)
)
client.socket.connect((ignite_host, ignite_port))
hs_request = HandshakeRequest(username, password)
client.send(hs_request)
hs_response = read_response(client)
assert hs_response['op_code'] == 0
client.close()
