def test_auth_failed(username, password, with_ssl, ssl_params): ssl_params['use_ssl'] = with_ssl with pytest.raises(AuthenticationError): client = Client(username=username, password=password, **ssl_params) with client.connect("127.0.0.1", 10801): pass
def test_auth_success(with_ssl, ssl_params): ssl_params['use_ssl'] = with_ssl client = Client(username=DEFAULT_IGNITE_USERNAME, password=DEFAULT_IGNITE_PASSWORD, **ssl_params) with client.connect("127.0.0.1", 10801): assert all(node.alive for node in client._nodes)
def inner(): client = Client(**kwargs) with client.connect("127.0.0.1", 10801): with get_or_create_cache(client, 'test-cache') as cache: cache.put(1, 1) assert cache.get(1) == 1
def __init__(self, url='127.0.0.1', port=10800): if ("IGNITE_CLUSTER_IP" in os.environ): url = os.environ["IGNITE_CLUSTER_IP"] if ("IGNITE_CLUSTER_PORT" in os.environ): port = int(os.environ["IGNITE_CLUSTER_PORT"]) print(url, port) self.client = Client() self.client.connect(url, port)
def get_data(self) -> list: client = Client() client.connect('localhost', 10800) streams_cache = client.get_or_create_cache( json.loads(self.__stream_name)['Stream']) stream = streams_cache.scan() return list(stream)
def skip_if_no_affinity(request, server1): client = Client(partition_aware=True) client.connect('127.0.0.1', 10801) if not client.partition_awareness_supported_by_protocol: pytest.skip( f'skipped {request.node.name}, partition awareness is not supported.' )
def main(): client = Client() client.connect('127.0.0.1', 10800) my_cache = client.get_or_create_cache('my cache') my_cache.put('my key', 42) result = my_cache.get('my key') print('my key: {}'.format(result))
class IgniteStorage(LightStorage): """ Apache Ignite storage for Light Requests and Responses. :param host: Ignite service hostname or IP address. :param port: Ignite service port. :param request_cache_name: The cache where LightRequests are stored. :param response_cache_name: The cache where LightResponses are stored. :param timeout: Timeout (in seconds) for socket operations. """ def __init__(self, host: str, port: int, request_cache_name: str, response_cache_name: str, timeout: int = 30): self.host = host self.port = port self.request_cache_name = request_cache_name self.response_cache_name = response_cache_name self.timeout = timeout self._client = None # type: Optional[Client] def get_cache(self, cache_name: str) -> Cache: """Get an Ignite Cache.""" if self._client is None: self._client = Client(timeout=self.timeout) self._client.connect(self.host, self.port) return self._client.get_cache(cache_name) def pop_light_request(self, uid: str) -> Optional[LightRequest]: """Look up a LightRequest by a unique id and then remove it.""" data = self.get_cache(self.request_cache_name).get_and_remove(uid) LOGGER.debug('Got Light Request from cache: id=%r, data=%s', uid, data) return LightRequest().load_xml( parse_xml(data)) if data is not None else None def pop_light_response(self, uid: str) -> Optional[LightResponse]: """Look up a LightResponse by a unique id and then remove it.""" data = self.get_cache(self.response_cache_name).get_and_remove(uid) LOGGER.debug('Got Light Response from cache: id=%r, data=%s', uid, data) return LightResponse().load_xml( parse_xml(data)) if data is not None else None def put_light_request(self, uid: str, request: LightRequest) -> None: """Store a LightRequest under a unique id.""" data = dump_xml(request.export_xml()).decode('utf-8') LOGGER.debug('Store Light Request to cache: id=%r, data=%s', uid, data) self.get_cache(self.request_cache_name).put(uid, data) def put_light_response(self, uid: str, response: LightResponse) -> None: """Store a LightResponse under a unique id.""" data = dump_xml(response.export_xml()).decode('utf-8') LOGGER.debug('Store Light Response to cache: id=%r, data=%s', uid, data) self.get_cache(self.response_cache_name).put(uid, data)
def main(): client = Client() client.connect('127.0.0.1', 10800) query = '' result = client.sql(query) for row in result: print(row)
class Database: def __init__(self, url='127.0.0.1', port=10800): self.client = Client() self.client.connect(url, port) def getClient(self): return self.client def closeClient(self, client): client.close()
def expiry_policy_supported(request, server1): client = Client() with client.connect('127.0.0.1', 10801): result = client.protocol_context.is_expiry_policy_supported() if not result and request.node.get_closest_marker( 'skip_if_no_expiry_policy'): pytest.skip( f'skipped {request.node.name}, ExpiryPolicy APIis not supported.' ) return result
class AuxiliaryIgniteStorage(AuxiliaryStorage): """ Apache Ignite storage for auxiliary data. :param host: Ignite service hostname or IP address. :param port: Ignite service port. :param cache_name: The cache where data are stored. :param timeout: Timeout (in seconds) for socket operations. """ def __init__(self, host: str, port: int, cache_name: str, timeout: int = 30, prefix: str = None): self.host = host self.port = port self.cache_name = cache_name self.timeout = timeout self.prefix = prefix self._client = None # type: Optional[Client] def get_cache(self, cache_name: str) -> Cache: """Get an Ignite Cache.""" if self._client is None: self._client = Client(timeout=self.timeout) self._client.connect(self.host, self.port) return self._client.get_cache(cache_name) def pop(self, uid: str) -> Optional[Dict[str, Any]]: """Look up data by a unique id and then remove it.""" if self.prefix: uid = self.prefix + uid data = self.get_cache(self.cache_name).get_and_remove(uid) LOGGER.debug('Got data from cache: id=%r, data=%s', uid, data) return json.loads(data) if data is not None else None def put(self, uid: str, data: Dict[str, Any]) -> None: """ Store data under a unique id. Data must be JSON-serializable. """ if self.prefix: uid = self.prefix + uid LOGGER.debug('Store data to cache: id=%r, data=%s', uid, data) self.get_cache(self.cache_name).put(uid, json.dumps(data, sort_keys=True))
class Database: def __init__(self, url='127.0.0.1', port=10800): if ("IGNITE_CLUSTER_IP" in os.environ): url = os.environ["IGNITE_CLUSTER_IP"] if ("IGNITE_CLUSTER_PORT" in os.environ): port = int(os.environ["IGNITE_CLUSTER_PORT"]) print(url, port) self.client = Client() self.client.connect(url, port) def getClient(self): return self.client def closeClient(self, client): client.close()
def artistMaster(request): loginUser = request.user nodes = [ ('127.0.0.1', 10800), ('127.0.0.1', 10801), ] client = Client() client.connect(nodes) #QUERY = ''' SELECT ID, TRACKID, TRACKDATA FROM PUBLIC.SONG_DATA ORDER BY ID DESC; ''' QUERY = ''' SELECT TRACKNAME, ALBUMNAME, ARTISTNAME, GENRE FROM PUBLIC.SONG_DATA ORDER BY ID DESC; ''' #print(QUERY) result = client.sql( QUERY, ) datatohtml = [] '''for row in result: try: dataparsed = json.loads(row[2]) try: genre = dataparsed.get("primary_genres").get("music_genre_list")[0].get("music_genre").get("music_genre_name") except Exception as e: genre = "-" dataauto ={ "track_name":dataparsed.get("track_name"), "artist_name":dataparsed.get("artist_name"), "album_name":dataparsed.get("album_name"), "genre":genre } datatohtml.append(dataauto) except Exception as e: pass # print(datatohtml)''' for row in result: dataauto = { "track_name":row[0], "artist_name":row[2], "album_name":row[1], "genre":row[3], } datatohtml.append(dataauto) return render(request,"ArtistMaster.html",{"loginUser":loginUser,"datatohtml":datatohtml})
def test_client_with_failed_server(request, with_partition_awareness): srv = start_ignite(idx=4) try: client = Client(partition_aware=with_partition_awareness) with client.connect([("127.0.0.1", 10804)]): cache = client.get_or_create_cache(request.node.name) cache.put(1, 1) kill_process_tree(srv.pid) if with_partition_awareness: ex_class = (ReconnectError, ConnectionResetError) else: ex_class = ConnectionResetError with pytest.raises(ex_class): cache.get(1) finally: kill_process_tree(srv.pid)
def main(): print("Running sync ExpiryPolicy example.") client = Client() with client.connect('127.0.0.1', 10800): print("Create cache with expiry policy.") try: ttl_cache = client.create_cache({ PROP_NAME: 'test', PROP_EXPIRY_POLICY: ExpiryPolicy(create=1.0) }) except NotSupportedByClusterError: print( "'ExpiryPolicy' API is not supported by cluster. Finishing...") return try: ttl_cache.put(1, 1) time.sleep(0.5) print(f"key = {1}, value = {ttl_cache.get(1)}") # key = 1, value = 1 time.sleep(1.2) print(f"key = {1}, value = {ttl_cache.get(1)}") # key = 1, value = None finally: ttl_cache.destroy() print("Create simple Cache and set TTL through `with_expire_policy`") simple_cache = client.create_cache('test') try: ttl_cache = simple_cache.with_expire_policy(access=1.0) ttl_cache.put(1, 1) time.sleep(0.5) print(f"key = {1}, value = {ttl_cache.get(1)}") # key = 1, value = 1 time.sleep(1.7) print(f"key = {1}, value = {ttl_cache.get(1)}") # key = 1, value = None finally: simple_cache.destroy()
def client(connection_param): client = Client(partition_aware=True, timeout=CLIENT_SOCKET_TIMEOUT) try: client.connect(connection_param) yield client finally: client.close()
def client(): client = Client() try: client.connect('127.0.0.1', 10801) yield client finally: client.close()
def client_routed(): client = Client(partition_aware=True) try: client.connect(client_routed_connection_string) yield client finally: client.close()
def client( ignite_host, ignite_port, timeout, use_ssl, ssl_keyfile, ssl_certfile, ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version, username, password, ): client = Client( timeout=timeout, use_ssl=use_ssl, ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile, ssl_ca_certfile=ssl_ca_certfile, ssl_cert_reqs=ssl_cert_reqs, ssl_ciphers=ssl_ciphers, ssl_version=ssl_version, username=username, password=password, ) client.connect(ignite_host, ignite_port) yield client for cache_name in cache_get_names(client).value: cache_destroy(client, cache_name) client.close()
def export_users_csv(request): print("Hello") nodes = [ ('127.0.0.1', 10800), ('127.0.0.1', 10801), ] client = Client() client.connect(nodes) yesterday = date.today() - timedelta(days=1) datecsv = yesterday.strftime('%m/%d/%Y') print(datecsv) QUERY = ''' SELECT ID, APPID, EVENTS, DEVICE_ID, "TIMESTAMP", MTIMESTAMP, UPTIMESTAMP, IPADDRESS, CITY, COUNTRY, EVENTNAME, CREATED_DATE, CREATED_TIME FROM PUBLIC.EVENTSDATA WHERE APPID != '5def994ce96b09565e1f1ddd' AND CREATED_DATE = {} AND EVENTNAME!='_app_crash' AND ID!=855263 ORDER BY ID; '''.format("'"+datecsv+"'") print(QUERY) result = client.sql( QUERY, include_field_names=True, ) print(next(result)) response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename="{}.csv"'.format(datecsv+"_IgniteEvents") writer = csv.writer(response) writer.writerow(['appid','deviceid','devicemodel','platform','apppackage','keyname','mobileoperator','app','song','album','pstate','source','ipaddress','city','station','duration','timestamp','to_char','created_time']) i = 0 for row in result: if i == 0: print(row) x = row[2] y = json.loads(x) if y.get("segmentation") is None: writer.writerow([row[1],row[3],y.get("d"),y.get("p"),y.get("ap"),row[10],y.get("network").get("ope"),"","","","","",row[7],row[8],"","",row[4],row[11],row[12]]) else: writer.writerow([row[1],row[3],y.get("d"),y.get("p"),y.get("ap"),row[10],y.get("network").get("ope"),y.get("segmentation").get("App"),y.get("segmentation").get("Song"),y.get("segmentation").get("Album"),y.get("segmentation").get("PState"),y.get("segmentation").get("Source"),row[7],row[8],y.get("segmentation").get("Station"),y.get("segmentation").get("Duration"),row[4],row[11],row[12]]) i = 1 else: print(row) x = row[5] try: y = json.loads(x) except: pass if y.get("segmentation") is None: writer.writerow([row[1],row[6],y.get("d"),y.get("p"),y.get("ap"),row[2],y.get("network").get("ope"),"","","","","",row[10],row[11],"","",row[7],row[3],row[4]]) else: writer.writerow([row[1],row[6],y.get("d"),y.get("p"),y.get("ap"),row[2],y.get("network").get("ope"),y.get("segmentation").get("App"),y.get("segmentation").get("Song"),y.get("segmentation").get("Album"),y.get("segmentation").get("PState"),y.get("segmentation").get("Source"),row[10],row[11],y.get("segmentation").get("Station"),y.get("segmentation").get("Duration"),row[7],row[3],row[4]]) client.close() return response
def test_cluster_set_active(with_persistence): key = 42 val = 42 start_state = ClusterState.INACTIVE if with_persistence else ClusterState.ACTIVE client = Client() with client.connect([("127.0.0.1", 10801), ("127.0.0.1", 10802)]): cluster = client.get_cluster() assert cluster.get_state() == start_state cluster.set_state(ClusterState.ACTIVE) assert cluster.get_state() == ClusterState.ACTIVE cache = client.get_or_create_cache("test_cache") cache.put(key, val) assert cache.get(key) == val cluster.set_state(ClusterState.ACTIVE_READ_ONLY) assert cluster.get_state() == ClusterState.ACTIVE_READ_ONLY assert cache.get(key) == val with pytest.raises(CacheError): cache.put(key, val + 1) cluster.set_state(ClusterState.INACTIVE) assert cluster.get_state() == ClusterState.INACTIVE with pytest.raises(CacheError): cache.get(key) with pytest.raises(CacheError): cache.put(key, val + 1) cluster.set_state(ClusterState.ACTIVE) assert cluster.get_state() == ClusterState.ACTIVE cache.put(key, val + 2) assert cache.get(key) == val + 2
def test_client_with_recovered_server(request, with_partition_awareness): srv = start_ignite(idx=4) try: client = Client(partition_aware=with_partition_awareness, timeout=CLIENT_SOCKET_TIMEOUT) with client.connect([("127.0.0.1", 10804)]): cache = client.get_or_create_cache(request.node.name) cache.put(1, 1) # Kill and restart server kill_process_tree(srv.pid) srv = start_ignite(idx=4) # First request may fail. try: cache.put(1, 2) except connection_errors: pass # Retry succeeds cache.put(1, 2) assert cache.get(1) == 2 finally: kill_process_tree(srv.pid)
def test_connection_context(connection_param, partition_aware): is_partition_aware = partition_aware == 'with_partition_aware' client = Client(partition_aware=is_partition_aware) # Check context manager with client.connect(connection_param): __check_open(client, is_partition_aware) __check_closed(client) # Check standard way try: client.connect(connection_param) __check_open(client, is_partition_aware) finally: client.close() __check_closed(client)
def test_connection_error_with_incorrect_config(invalid_ssl_params): with pytest.raises(ReconnectError): client = Client(**invalid_ssl_params) with client.connect([("127.0.0.1", 10801)]): pass
from pyignite import Client import ssl #tag::no-ssl[] client = Client(username='******', password='******', use_ssl=False) #end::no-ssl[] client = Client( ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_keyfile='/path/to/key/file', ssl_certfile='/path/to/client/cert', ssl_ca_certfile='/path/to/trusted/cert/or/chain', username='******', password='******', ) client.connect('localhost', 10800)
def test_handshake( monkeypatch, ignite_host, ignite_port, use_ssl, ssl_keyfile, ssl_certfile, ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version, username, password, ): client = Client( use_ssl=use_ssl, ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile, ssl_ca_certfile=ssl_ca_certfile, ssl_cert_reqs=ssl_cert_reqs, ssl_ciphers=ssl_ciphers, ssl_version=ssl_version, username=username, password=password, ) client._socket = client._wrap( socket.socket(socket.AF_INET, socket.SOCK_STREAM)) client.socket.connect((ignite_host, ignite_port)) hs_request = HandshakeRequest(username, password) client.send(hs_request) hs_response = read_response(client) assert hs_response['op_code'] != 0 client.close() # intentionally pass wrong protocol version from pyignite.connection import handshake monkeypatch.setattr(handshake, 'PROTOCOL_VERSION_MAJOR', 10) client._socket = client._wrap( socket.socket(socket.AF_INET, socket.SOCK_STREAM)) client.socket.connect((ignite_host, ignite_port)) hs_request = HandshakeRequest(username, password) client.send(hs_request) hs_response = read_response(client) assert hs_response['op_code'] == 0 client.close()
# # https://www.gridgain.com/products/software/community-edition/gridgain-community-edition-license # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # from collections import OrderedDict from pyignite import Client, GenericObjectMeta from pyignite.datatypes import DoubleObject, IntObject, String from pyignite.datatypes.prop_codes import * client = Client() client.connect('127.0.0.1', 10800) student_cache = client.create_cache({ PROP_NAME: 'SQL_PUBLIC_STUDENT', PROP_SQL_SCHEMA: 'PUBLIC', PROP_QUERY_ENTITIES: [ { 'table_name': 'Student'.upper(), 'key_field_name': 'SID', 'key_type_name': 'java.lang.Integer', 'field_name_aliases': [], 'query_fields': [ { 'name': 'SID', 'type_name': 'java.lang.Integer',
from collections import OrderedDict from pyignite import Client, GenericObjectMeta from pyignite.datatypes import * class Person(metaclass=GenericObjectMeta, schema=OrderedDict([ ('first_name', String), ('last_name', String), ('age', IntObject), ])): pass client = Client() client.connect('localhost', 10800) person_cache = client.get_or_create_cache('person') person_cache.put( 1, Person(first_name='Ivan', last_name='Ivanov', age=33) ) person = person_cache.get(1) print(person.__class__.__name__) # Person print(person.__class__ is Person) # True if `Person` was registered automatically (on writing) # or manually (using `client.register_binary_type()` method).
from pyignite import Client client = Client() client.connect('127.0.0.1', 10800) PRODUCT_CREATE_TABLE = ''' create table product( product_id VARCHAR PRIMARY KEY, product_url VARCHAR ) ''' client.sql(PRODUCT_CREATE_TABLE) SIMILARPRODUCTS_CREATE_TABLE = ''' create table similarproducts( product_id VARCHAR , similar_product_id VARCHAR , similariy_score DOUBLE , PRIMARY KEY (product_id, similar_product_id) ) WITH "affinityKey=similar_product_id" ''' client.sql(SIMILARPRODUCTS_CREATE_TABLE) PRODUCT_INSERT_QUERY = ''' insert into product values (? , ?) ''' SIMILARPRODUCTS_INSERT_QUERY = ''' insert into similarproducts values (? , ?, ?) ''' # Code for inserting dummy data INSERT_PRODUCT_DATA= [] for i in range(10000) :
# See the License for the specific language governing permissions and # limitations under the License. from pyignite import Client from pyignite.datatypes.cache_config import CacheMode from pyignite.datatypes.prop_codes import * from pyignite.exceptions import SocketError nodes = [ ('127.0.0.1', 10800), ('127.0.0.1', 10801), ('127.0.0.1', 10802), ] client = Client(timeout=4.0) client.connect(nodes) print('Connected to {}'.format(client)) my_cache = client.get_or_create_cache({ PROP_NAME: 'my_cache', PROP_CACHE_MODE: CacheMode.REPLICATED, }) my_cache.put('test_key', 0) # abstract main loop while True: try: # do the work test_value = my_cache.get('test_key') my_cache.put('test_key', test_value + 1)
# include, and the License does not grant to you, the right to Sell the Software. # For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you # under the License to provide to third parties, for a fee or other consideration (including without # limitation fees for hosting or consulting/ support services related to the Software), a product or # service whose value derives, entirely or substantially, from the functionality of the Software. # Any license notice or attribution required by the License must also include this Commons Clause # License Condition notice. # # For purposes of the clause above, the “Licensor” is Copyright 2019 GridGain Systems, Inc., # the “License” is the Apache License, Version 2.0, and the Software is the GridGain Community # Edition software provided with this notice. from pyignite import Client from pyignite.datatypes import CharObject, ShortObject client = Client() client.connect('127.0.0.1', 10800) my_cache = client.get_or_create_cache('my cache') my_cache.put('my key', 42) # value ‘42’ takes 9 bytes of memory as a LongObject my_cache.put('my key', 42, value_hint=ShortObject) # value ‘42’ takes only 3 bytes as a ShortObject my_cache.put('a', 1) # ‘a’ is a key of type String my_cache.put('a', 2, key_hint=CharObject) # another key ‘a’ of type CharObject was created
# 'report_date': DateObject, # 'purpose': String, # 'sum': DecimalObject, # 'recipient': String, # 'cashier_id': LongObject, # } class ExpenseVoucher( metaclass=GenericObjectMeta, schema=old_schema, ): pass client = Client() client.connect('127.0.0.1', 10800) accounting = client.get_or_create_cache('accounting') for key, value in old_data: accounting.put(key, ExpenseVoucher(**value)) data_classes = client.query_binary_type('ExpenseVoucher') print(data_classes) # { # -231598180: <class '__main__.ExpenseVoucher'> # } s_id, data_class = data_classes.popitem() schema = data_class.schema
# The ASF licenses this file to You under the Apache License, Version 2.0 # (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from pyignite import Client from pyignite.datatypes import CharObject, ShortObject client = Client() client.connect('127.0.0.1', 10800) my_cache = client.get_or_create_cache('my cache') my_cache.put('my key', 42) # value ‘42’ takes 9 bytes of memory as a LongObject my_cache.put('my key', 42, value_hint=ShortObject) # value ‘42’ takes only 3 bytes as a ShortObject my_cache.put('a', 1) # ‘a’ is a key of type String my_cache.put('a', 2, key_hint=CharObject) # another key ‘a’ of type CharObject was created
['CHN', 'Dong', False, Decimal('0.2')], ['CHN', 'Hui', False, Decimal('0.8')], ['CHN', 'Mantšu', False, Decimal('0.9')], ['CHN', 'Miao', False, Decimal('0.7')], ['CHN', 'Mongolian', False, Decimal('0.4')], ['CHN', 'Puyi', False, Decimal('0.2')], ['CHN', 'Tibetan', False, Decimal('0.4')], ['CHN', 'Tujia', False, Decimal('0.5')], ['CHN', 'Uighur', False, Decimal('0.6')], ['CHN', 'Yi', False, Decimal('0.6')], ['CHN', 'Zhuang', False, Decimal('1.4')], ] # establish connection client = Client() client.connect('127.0.0.1', 10800) # create tables for query in [ COUNTRY_CREATE_TABLE_QUERY, CITY_CREATE_TABLE_QUERY, LANGUAGE_CREATE_TABLE_QUERY, ]: client.sql(query) # create indices for query in [CITY_CREATE_INDEX, LANGUAGE_CREATE_INDEX]: client.sql(query) # load data
def export_users_csv(): print("Hello") nodes = [ ('35.154.247.92', 10800), ('35.154.247.92', 10801), ] conn = psycopg2.connect( dbname="VTIONData", host="vtionproddb.chgz4nqwpdta.ap-south-1.rds.amazonaws.com", user="******", password="******") cur = conn.cursor() client = Client() client.connect(nodes) yesterday = date.today() - timedelta(days=1) yesterdayMidnight = int(time.mktime(yesterday.timetuple())) - 19800 print("FROM : ", yesterdayMidnight) todayMidnight = yesterdayMidnight + 86400 print(todayMidnight) yesterdayMidnight = '1579199400' todayMidnight = '1579285800' QUERY = ''' SELECT ID, APPID, EVENTS, DEVICE_ID, "TIMESTAMP", MTIMESTAMP, UPTIMESTAMP, IPADDRESS, CITY, COUNTRY, EVENTNAME, CREATED_DATE, CREATED_TIME FROM PUBLIC.EVENTSDATA WHERE "TIMESTAMP" > '{}' AND "TIMESTAMP" < '{}' ;'''.format( yesterdayMidnight, todayMidnight) result = client.sql( QUERY, include_field_names=True, ) next(result) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) print(BASE_DIR) namespac = BASE_DIR + str("/target/Data_" + yesterday.strftime('%m_%d_%Y') + '.csv') with open(namespac, 'w+') as writeFile: print("START : ", time.localtime(time.time())) writer = csv.writer(writeFile) writer.writerow([ 'appid', 'DeviceId', 'vtionid', 'deviceModel', 'platform', 'apppackage', 'keyname', 'mobileOperator', 'app', 'song', 'album', 'Pstate', 'Program', 'Episode', 'Source', 'ipaddress', 'city', 'station', 'duration', 'timestamp', 'created_date', 'created_time', 'stationName', 'artist_name', 'genre', 'education', 'ownership', 'nccs_code', 'age', 'gender', 'number', 'Uninstall', 'Reward_option' ]) i = 0 for row in result: nccs = age = gender = number = status = '' artist_name = '' genre = '' try: cur.execute( '''SELECT nccs_code FROM public.installdata WHERE deviceid = '{}';''' .format(row[6])) nccs_result = cur.fetchone() nccs_res = nccs_result[0] except Exception as e: nccs_res = '' pass if i == 0: x = row[2] i = 1 else: x = row[5] urlSafeEncodedBytes = base64.urlsafe_b64encode( row[6].encode("utf-8")) vtionid = str(urlSafeEncodedBytes, "utf-8") try: y = json.loads(x) if y.get("segmentation") is None: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), "", "", "", "", "", "", "", row[10], row[11], "", "", row[7], row[3], row[4] ]) else: if y.get("ap") == "com.video.meter": song = y.get("segmentation").get("Song") app = y.get("segmentation").get("App") education = y.get("segmentation").get( "Highest Education") ownership = y.get("segmentation").get("Ownership") if row[2] == 'FM_Tuned': try: QUERYa = ''' SELECT STATION FROM MASTERTABLEDEMO WHERE CITY = '{}' AND FREQUENCY = '{}'; '''.format( row[11], y.get("segmentation").get("Station")) resulta = client.sql( QUERYa, include_field_names=True, ) next(resulta) stationName = [rowa for rowa in resulta] writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get("PState"), "", "", y.get("segmentation").get("Source"), row[10], row[11], y.get("segmentation").get("Station"), y.get("segmentation").get("Duration"), row[7], row[3], row[4], stationName[0][0], "", "", "", "", nccs_res ]) except Exception as e: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get("PState"), "", "", y.get("segmentation").get("Source"), row[10], row[11], y.get("segmentation").get("Station"), y.get("segmentation").get("Duration"), row[7], row[3], row[4], "", "", "", "", "", nccs_res ]) elif row[2] == 'Video_Tuned' or row[ 2] == 'Video_off': try: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get("PState"), y.get("segmentation").get("Program"), y.get("segmentation").get("Episode"), y.get("segmentation").get("Source"), row[10], row[11], "", y.get("segmentation").get("Duration"), row[7], row[3], row[4], "", "", "", "", "", nccs_res ]) except Exception as e: pass elif row[2] == 'Audio_Tuned': try: if song: if song.isdigit(): continue song = song.strip() filter_new = [ 'Download', 'download', 'AUD', 'Advertise', '%' ] status_filter = ( filter_new[0] in song or filter_new[1] in song or filter_new[2] in song or filter_new[3] in song or filter_new[4] in song) if str(status_filter) == 'False': album = y.get("segmentation").get( "Album") if album is None: album_filter = False else: a_filter = [ 'unknown', 'Advertise', 'Sponsored' ] album_filter = ( a_filter[0] in album or a_filter[1] in album or a_filter[2] in album) if str(album_filter) == 'False': QUERY = '''SELECT ARTISTNAME, GENRE FROM PUBLIC.SONG_DATA WHERE TRACKNAME = '{}';'''.format( song) details = client.sql( QUERY, include_field_names=True, ) next(details) for det in details: artist_name = det[0] genre = det[1] if artist_name: pass else: QUERY = '''SELECT ARTISTNAME, GENRE FROM PUBLIC.SONG_DATA_ADD WHERE TRACKNAME = '{}';'''.format( song) details = client.sql( QUERY, include_field_names= True, ) next(details) for det in details: artist_name = det[0] genre = det[1] if app == 'Amazon Music': pass if '%' in song or song.isdigit( ): continue else: if 'ñ' in song: song = song.replace( 'ñ', 'n') else: pass song = deEmojify( filter( remove_bdata(song). replace( ')', '').replace( '(', ''))) elif app == 'Spotify' or 'Spot' in app: if '%' in song: continue else: song = filter( deEmojify( remove_bdata( song))) elif app == 'Gaana': if 'Gaana' in song or song.isdigit( ) or song == ',': continue else: if 'ñ' in song or '├▒' in song: song = song.replace( 'ñ', 'n').replace( '├▒', 'n') else: pass song = select_before_list( remove_bdata( deEmojify(song)) ).lstrip('Ep 0123456789-.') elif app == 'Google Play Music': aa = song if song.isdigit( ) or 'high quality' in song or song == ',': continue else: song = domain( remove_bdata(song) ).lstrip( ' 0123456789.-' ).strip() if '128k' in song or '256k' in song or 'p3' in song: song = song.replace( '(128k)', '' ).replace( '(256k)', '').replace( '128k', '' ).replace( '256k', '' ).replace( '.mp3', '') song = select_before_list( filter( deEmojify( song))) elif app == 'Hungama': song = filter( remove_bdata( deEmojify(song))) elif app == 'i Music': song = song.replace( '-', ' ').replace( ':', ' ').replace( ' ', ' ').replace( ' ', ' ' ).replace( '.mp3', '') if 'Unknown artist' in song or 'i Music' in song or '//' in song: continue song = deEmojify( select_before_list( filter( domain( remove_bdata( song)) ).lstrip( ' 0123456789.-' ).strip())).strip( ) elif app == 'JioSaavn': song = filter( remove_bdata(song)) elif app == 'Saavn': #combine with jio savan song = filter( domain( remove_bdata( song))) app = 'JioSaavn' elif app == 'MX Player': if '--:--:--' in song or 'Insufficient balance' in song or 'Expiry Date' in song: continue else: song = deEmojify( filter( remove_bdata( song) ).replace( ' ', ' ')).strip() elif app == 'Music Player': song = domain( filter( remove_bdata(song)) ).replace( ' ', ' ').lstrip( ' 0123456789.-' ).strip() elif app == 'Wynk Music': song = remove_bdata( deEmojify(song) ).lstrip(' 0123456789.-' ).strip() if song.isdigit( ) or '%' in song: continue elif '(From' in song: song = song.replace( '(From', '') else: if '")' in song: song = remove_bdata( song.replace( '\")', '')) elif app == 'YouTube Music': song = filter( deEmojify( remove_bdata( song))) else: song = song app = app try: if song: if song.isdigit(): continue song = song.strip() writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network" ).get("ope"), app, song, y.get( "segmentation" ).get("Album"), y.get( "segmentation" ).get("PState"), "", "", y.get( "segmentation" ).get("Source"), row[10], row[11], y.get( "segmentation" ).get("Station"), y.get( "segmentation" ).get("Duration"), row[7], row[3], row[4], "", artist_name, genre, '', '', nccs_res, '', '', '', '' ]) else: pass except Exception as e: pass else: pass else: pass except Exception as e: pass elif row[2] == 'Register' or row[2] == 'Profile': if ownership: try: own = ownership.split(',') if len(own) >= 9: num_own = 9 else: num_own = len(own) cur.execute( '''SELECT nccs_code FROM public.nccs_flat where education = '{}' and ownership = '{}';'''. format(education, num_own)) nccs = cur.fetchone() except Exception as e: pass try: try: cur.execute( '''SELECT payment_option FROM public.payment_option where deviceid = '{}'; '''.format(row[6])) payment_option = cur.fetchone() payment = payment_option[0] except: payment = 'Amazon voucher' pass age = y.get("segmentation").get("age") if age: pass else: age = y.get("segmentation").get("Age") gender = y.get("segmentation").get( "Gender") if gender: pass else: gender = y.get("segmentation").get( "gender") number = y.get("segmentation").get( "Mobile Number") num_status = "+" in number if str(num_status) == 'True': number = '' else: try: cur.execute( '''SELECT i_status FROM public.appsflyer where number = '{}';'''.format( number)) status_now = cur.fetchone() status = status_now[0] if status != 'True': status = '' except: pass try: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get( "PState"), "", "", y.get("segmentation").get( "Source"), row[10], row[11], y.get("segmentation").get( "Station"), y.get("segmentation").get( "Duration"), row[7], row[3], row[4], "", artist_name, genre, education, ownership, nccs[0], age, gender, number, status, payment ]) except Exception as e: pass writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get( "PState"), "", "", y.get("segmentation").get( "Source"), row[10], row[11], y.get("segmentation").get( "Station"), y.get("segmentation").get( "Duration"), row[7], row[3], row[4], "", artist_name, genre, education, ownership, "", age, gender, number, status, payment ]) except Exception as e: pass else: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get("PState"), "", "", y.get("segmentation").get("Source"), row[10], row[11], y.get("segmentation").get("Station"), y.get("segmentation").get("Duration"), row[7], row[3], row[4], "", artist_name, genre, education, ownership, "" ]) except Exception as e: pass client.close() writeFile.close()
def test_handshake( monkeypatch, ignite_host, ignite_port, use_ssl, ssl_keyfile, ssl_certfile, ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version, username, password, ): client = Client( use_ssl=use_ssl, ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile, ssl_ca_certfile=ssl_ca_certfile, ssl_cert_reqs=ssl_cert_reqs, ssl_ciphers=ssl_ciphers, ssl_version=ssl_version, username=username, password=password, ) client._socket = client._wrap( socket.socket(socket.AF_INET, socket.SOCK_STREAM) ) client.socket.connect((ignite_host, ignite_port)) hs_request = HandshakeRequest(username, password) client.send(hs_request) hs_response = read_response(client) assert hs_response['op_code'] != 0 client.close() # intentionally pass wrong protocol version from pyignite.connection import handshake monkeypatch.setattr(handshake, 'PROTOCOL_VERSION_MAJOR', 10) client._socket = client._wrap( socket.socket(socket.AF_INET, socket.SOCK_STREAM) ) client.socket.connect((ignite_host, ignite_port)) hs_request = HandshakeRequest(username, password) client.send(hs_request) hs_response = read_response(client) assert hs_response['op_code'] == 0 client.close()