def client_routed(): client = Client(partition_aware=True) try: client.connect(client_routed_connection_string) yield client finally: client.close()
def client( ignite_host, ignite_port, timeout, use_ssl, ssl_keyfile, ssl_certfile, ssl_ca_certfile, ssl_cert_reqs, ssl_ciphers, ssl_version, username, password, ): client = Client( timeout=timeout, use_ssl=use_ssl, ssl_keyfile=ssl_keyfile, ssl_certfile=ssl_certfile, ssl_ca_certfile=ssl_ca_certfile, ssl_cert_reqs=ssl_cert_reqs, ssl_ciphers=ssl_ciphers, ssl_version=ssl_version, username=username, password=password, ) client.connect(ignite_host, ignite_port) yield client for cache_name in cache_get_names(client).value: cache_destroy(client, cache_name) client.close()
def client(): client = Client() try: client.connect('127.0.0.1', 10801) yield client finally: client.close()
def client(connection_param): client = Client(partition_aware=True, timeout=CLIENT_SOCKET_TIMEOUT) try: client.connect(connection_param) yield client finally: client.close()
def skip_if_no_affinity(request, server1): client = Client(partition_aware=True) client.connect('127.0.0.1', 10801) if not client.partition_awareness_supported_by_protocol: pytest.skip( f'skipped {request.node.name}, partition awareness is not supported.' )
def get_data(self) -> list: client = Client() client.connect('localhost', 10800) streams_cache = client.get_or_create_cache( json.loads(self.__stream_name)['Stream']) stream = streams_cache.scan() return list(stream)
def main(): client = Client() client.connect('127.0.0.1', 10800) my_cache = client.get_or_create_cache('my cache') my_cache.put('my key', 42) result = my_cache.get('my key') print('my key: {}'.format(result))
class IgniteStorage(LightStorage): """ Apache Ignite storage for Light Requests and Responses. :param host: Ignite service hostname or IP address. :param port: Ignite service port. :param request_cache_name: The cache where LightRequests are stored. :param response_cache_name: The cache where LightResponses are stored. :param timeout: Timeout (in seconds) for socket operations. """ def __init__(self, host: str, port: int, request_cache_name: str, response_cache_name: str, timeout: int = 30): self.host = host self.port = port self.request_cache_name = request_cache_name self.response_cache_name = response_cache_name self.timeout = timeout self._client = None # type: Optional[Client] def get_cache(self, cache_name: str) -> Cache: """Get an Ignite Cache.""" if self._client is None: self._client = Client(timeout=self.timeout) self._client.connect(self.host, self.port) return self._client.get_cache(cache_name) def pop_light_request(self, uid: str) -> Optional[LightRequest]: """Look up a LightRequest by a unique id and then remove it.""" data = self.get_cache(self.request_cache_name).get_and_remove(uid) LOGGER.debug('Got Light Request from cache: id=%r, data=%s', uid, data) return LightRequest().load_xml( parse_xml(data)) if data is not None else None def pop_light_response(self, uid: str) -> Optional[LightResponse]: """Look up a LightResponse by a unique id and then remove it.""" data = self.get_cache(self.response_cache_name).get_and_remove(uid) LOGGER.debug('Got Light Response from cache: id=%r, data=%s', uid, data) return LightResponse().load_xml( parse_xml(data)) if data is not None else None def put_light_request(self, uid: str, request: LightRequest) -> None: """Store a LightRequest under a unique id.""" data = dump_xml(request.export_xml()).decode('utf-8') LOGGER.debug('Store Light Request to cache: id=%r, data=%s', uid, data) self.get_cache(self.request_cache_name).put(uid, data) def put_light_response(self, uid: str, response: LightResponse) -> None: """Store a LightResponse under a unique id.""" data = dump_xml(response.export_xml()).decode('utf-8') LOGGER.debug('Store Light Response to cache: id=%r, data=%s', uid, data) self.get_cache(self.response_cache_name).put(uid, data)
def main(): client = Client() client.connect('127.0.0.1', 10800) query = '' result = client.sql(query) for row in result: print(row)
class Database: def __init__(self, url='127.0.0.1', port=10800): self.client = Client() self.client.connect(url, port) def getClient(self): return self.client def closeClient(self, client): client.close()
def export_users_csv(request): print("Hello") nodes = [ ('127.0.0.1', 10800), ('127.0.0.1', 10801), ] client = Client() client.connect(nodes) yesterday = date.today() - timedelta(days=1) datecsv = yesterday.strftime('%m/%d/%Y') print(datecsv) QUERY = ''' SELECT ID, APPID, EVENTS, DEVICE_ID, "TIMESTAMP", MTIMESTAMP, UPTIMESTAMP, IPADDRESS, CITY, COUNTRY, EVENTNAME, CREATED_DATE, CREATED_TIME FROM PUBLIC.EVENTSDATA WHERE APPID != '5def994ce96b09565e1f1ddd' AND CREATED_DATE = {} AND EVENTNAME!='_app_crash' AND ID!=855263 ORDER BY ID; '''.format("'"+datecsv+"'") print(QUERY) result = client.sql( QUERY, include_field_names=True, ) print(next(result)) response = HttpResponse(content_type='text/csv') response['Content-Disposition'] = 'attachment; filename="{}.csv"'.format(datecsv+"_IgniteEvents") writer = csv.writer(response) writer.writerow(['appid','deviceid','devicemodel','platform','apppackage','keyname','mobileoperator','app','song','album','pstate','source','ipaddress','city','station','duration','timestamp','to_char','created_time']) i = 0 for row in result: if i == 0: print(row) x = row[2] y = json.loads(x) if y.get("segmentation") is None: writer.writerow([row[1],row[3],y.get("d"),y.get("p"),y.get("ap"),row[10],y.get("network").get("ope"),"","","","","",row[7],row[8],"","",row[4],row[11],row[12]]) else: writer.writerow([row[1],row[3],y.get("d"),y.get("p"),y.get("ap"),row[10],y.get("network").get("ope"),y.get("segmentation").get("App"),y.get("segmentation").get("Song"),y.get("segmentation").get("Album"),y.get("segmentation").get("PState"),y.get("segmentation").get("Source"),row[7],row[8],y.get("segmentation").get("Station"),y.get("segmentation").get("Duration"),row[4],row[11],row[12]]) i = 1 else: print(row) x = row[5] try: y = json.loads(x) except: pass if y.get("segmentation") is None: writer.writerow([row[1],row[6],y.get("d"),y.get("p"),y.get("ap"),row[2],y.get("network").get("ope"),"","","","","",row[10],row[11],"","",row[7],row[3],row[4]]) else: writer.writerow([row[1],row[6],y.get("d"),y.get("p"),y.get("ap"),row[2],y.get("network").get("ope"),y.get("segmentation").get("App"),y.get("segmentation").get("Song"),y.get("segmentation").get("Album"),y.get("segmentation").get("PState"),y.get("segmentation").get("Source"),row[10],row[11],y.get("segmentation").get("Station"),y.get("segmentation").get("Duration"),row[7],row[3],row[4]]) client.close() return response
class AuxiliaryIgniteStorage(AuxiliaryStorage): """ Apache Ignite storage for auxiliary data. :param host: Ignite service hostname or IP address. :param port: Ignite service port. :param cache_name: The cache where data are stored. :param timeout: Timeout (in seconds) for socket operations. """ def __init__(self, host: str, port: int, cache_name: str, timeout: int = 30, prefix: str = None): self.host = host self.port = port self.cache_name = cache_name self.timeout = timeout self.prefix = prefix self._client = None # type: Optional[Client] def get_cache(self, cache_name: str) -> Cache: """Get an Ignite Cache.""" if self._client is None: self._client = Client(timeout=self.timeout) self._client.connect(self.host, self.port) return self._client.get_cache(cache_name) def pop(self, uid: str) -> Optional[Dict[str, Any]]: """Look up data by a unique id and then remove it.""" if self.prefix: uid = self.prefix + uid data = self.get_cache(self.cache_name).get_and_remove(uid) LOGGER.debug('Got data from cache: id=%r, data=%s', uid, data) return json.loads(data) if data is not None else None def put(self, uid: str, data: Dict[str, Any]) -> None: """ Store data under a unique id. Data must be JSON-serializable. """ if self.prefix: uid = self.prefix + uid LOGGER.debug('Store data to cache: id=%r, data=%s', uid, data) self.get_cache(self.cache_name).put(uid, json.dumps(data, sort_keys=True))
class Database: def __init__(self, url='127.0.0.1', port=10800): if ("IGNITE_CLUSTER_IP" in os.environ): url = os.environ["IGNITE_CLUSTER_IP"] if ("IGNITE_CLUSTER_PORT" in os.environ): port = int(os.environ["IGNITE_CLUSTER_PORT"]) print(url, port) self.client = Client() self.client.connect(url, port) def getClient(self): return self.client def closeClient(self, client): client.close()
def test_auth_success(with_ssl, ssl_params): ssl_params['use_ssl'] = with_ssl client = Client(username=DEFAULT_IGNITE_USERNAME, password=DEFAULT_IGNITE_PASSWORD, **ssl_params) with client.connect("127.0.0.1", 10801): assert all(node.alive for node in client._nodes)
def test_auth_failed(username, password, with_ssl, ssl_params): ssl_params['use_ssl'] = with_ssl with pytest.raises(AuthenticationError): client = Client(username=username, password=password, **ssl_params) with client.connect("127.0.0.1", 10801): pass
def inner(): client = Client(**kwargs) with client.connect("127.0.0.1", 10801): with get_or_create_cache(client, 'test-cache') as cache: cache.put(1, 1) assert cache.get(1) == 1
def test_connection_context(connection_param, partition_aware): is_partition_aware = partition_aware == 'with_partition_aware' client = Client(partition_aware=is_partition_aware) # Check context manager with client.connect(connection_param): __check_open(client, is_partition_aware) __check_closed(client) # Check standard way try: client.connect(connection_param) __check_open(client, is_partition_aware) finally: client.close() __check_closed(client)
def artistMaster(request): loginUser = request.user nodes = [ ('127.0.0.1', 10800), ('127.0.0.1', 10801), ] client = Client() client.connect(nodes) #QUERY = ''' SELECT ID, TRACKID, TRACKDATA FROM PUBLIC.SONG_DATA ORDER BY ID DESC; ''' QUERY = ''' SELECT TRACKNAME, ALBUMNAME, ARTISTNAME, GENRE FROM PUBLIC.SONG_DATA ORDER BY ID DESC; ''' #print(QUERY) result = client.sql( QUERY, ) datatohtml = [] '''for row in result: try: dataparsed = json.loads(row[2]) try: genre = dataparsed.get("primary_genres").get("music_genre_list")[0].get("music_genre").get("music_genre_name") except Exception as e: genre = "-" dataauto ={ "track_name":dataparsed.get("track_name"), "artist_name":dataparsed.get("artist_name"), "album_name":dataparsed.get("album_name"), "genre":genre } datatohtml.append(dataauto) except Exception as e: pass # print(datatohtml)''' for row in result: dataauto = { "track_name":row[0], "artist_name":row[2], "album_name":row[1], "genre":row[3], } datatohtml.append(dataauto) return render(request,"ArtistMaster.html",{"loginUser":loginUser,"datatohtml":datatohtml})
def expiry_policy_supported(request, server1): client = Client() with client.connect('127.0.0.1', 10801): result = client.protocol_context.is_expiry_policy_supported() if not result and request.node.get_closest_marker( 'skip_if_no_expiry_policy'): pytest.skip( f'skipped {request.node.name}, ExpiryPolicy APIis not supported.' ) return result
def test_client_with_failed_server(request, with_partition_awareness): srv = start_ignite(idx=4) try: client = Client(partition_aware=with_partition_awareness) with client.connect([("127.0.0.1", 10804)]): cache = client.get_or_create_cache(request.node.name) cache.put(1, 1) kill_process_tree(srv.pid) if with_partition_awareness: ex_class = (ReconnectError, ConnectionResetError) else: ex_class = ConnectionResetError with pytest.raises(ex_class): cache.get(1) finally: kill_process_tree(srv.pid)
def main(): print("Running sync ExpiryPolicy example.") client = Client() with client.connect('127.0.0.1', 10800): print("Create cache with expiry policy.") try: ttl_cache = client.create_cache({ PROP_NAME: 'test', PROP_EXPIRY_POLICY: ExpiryPolicy(create=1.0) }) except NotSupportedByClusterError: print( "'ExpiryPolicy' API is not supported by cluster. Finishing...") return try: ttl_cache.put(1, 1) time.sleep(0.5) print(f"key = {1}, value = {ttl_cache.get(1)}") # key = 1, value = 1 time.sleep(1.2) print(f"key = {1}, value = {ttl_cache.get(1)}") # key = 1, value = None finally: ttl_cache.destroy() print("Create simple Cache and set TTL through `with_expire_policy`") simple_cache = client.create_cache('test') try: ttl_cache = simple_cache.with_expire_policy(access=1.0) ttl_cache.put(1, 1) time.sleep(0.5) print(f"key = {1}, value = {ttl_cache.get(1)}") # key = 1, value = 1 time.sleep(1.7) print(f"key = {1}, value = {ttl_cache.get(1)}") # key = 1, value = None finally: simple_cache.destroy()
def test_cluster_set_active(with_persistence): key = 42 val = 42 start_state = ClusterState.INACTIVE if with_persistence else ClusterState.ACTIVE client = Client() with client.connect([("127.0.0.1", 10801), ("127.0.0.1", 10802)]): cluster = client.get_cluster() assert cluster.get_state() == start_state cluster.set_state(ClusterState.ACTIVE) assert cluster.get_state() == ClusterState.ACTIVE cache = client.get_or_create_cache("test_cache") cache.put(key, val) assert cache.get(key) == val cluster.set_state(ClusterState.ACTIVE_READ_ONLY) assert cluster.get_state() == ClusterState.ACTIVE_READ_ONLY assert cache.get(key) == val with pytest.raises(CacheError): cache.put(key, val + 1) cluster.set_state(ClusterState.INACTIVE) assert cluster.get_state() == ClusterState.INACTIVE with pytest.raises(CacheError): cache.get(key) with pytest.raises(CacheError): cache.put(key, val + 1) cluster.set_state(ClusterState.ACTIVE) assert cluster.get_state() == ClusterState.ACTIVE cache.put(key, val + 2) assert cache.get(key) == val + 2
def test_client_with_recovered_server(request, with_partition_awareness): srv = start_ignite(idx=4) try: client = Client(partition_aware=with_partition_awareness, timeout=CLIENT_SOCKET_TIMEOUT) with client.connect([("127.0.0.1", 10804)]): cache = client.get_or_create_cache(request.node.name) cache.put(1, 1) # Kill and restart server kill_process_tree(srv.pid) srv = start_ignite(idx=4) # First request may fail. try: cache.put(1, 2) except connection_errors: pass # Retry succeeds cache.put(1, 2) assert cache.get(1) == 2 finally: kill_process_tree(srv.pid)
def test_connection_error_with_incorrect_config(invalid_ssl_params): with pytest.raises(ReconnectError): client = Client(**invalid_ssl_params) with client.connect([("127.0.0.1", 10801)]): pass
from pyignite import Client import ssl #tag::no-ssl[] client = Client(username='******', password='******', use_ssl=False) #end::no-ssl[] client = Client( ssl_cert_reqs=ssl.CERT_REQUIRED, ssl_keyfile='/path/to/key/file', ssl_certfile='/path/to/client/cert', ssl_ca_certfile='/path/to/trusted/cert/or/chain', username='******', password='******', ) client.connect('localhost', 10800)
# For purposes of the foregoing, “Sell” means practicing any or all of the rights granted to you # under the License to provide to third parties, for a fee or other consideration (including without # limitation fees for hosting or consulting/ support services related to the Software), a product or # service whose value derives, entirely or substantially, from the functionality of the Software. # Any license notice or attribution required by the License must also include this Commons Clause # License Condition notice. # # For purposes of the clause above, the “Licensor” is Copyright 2019 GridGain Systems, Inc., # the “License” is the Apache License, Version 2.0, and the Software is the GridGain Community # Edition software provided with this notice. from pyignite import Client from pyignite.datatypes import CharObject, ShortObject client = Client() client.connect('127.0.0.1', 10800) my_cache = client.get_or_create_cache('my cache') my_cache.put('my key', 42) # value ‘42’ takes 9 bytes of memory as a LongObject my_cache.put('my key', 42, value_hint=ShortObject) # value ‘42’ takes only 3 bytes as a ShortObject my_cache.put('a', 1) # ‘a’ is a key of type String my_cache.put('a', 2, key_hint=CharObject) # another key ‘a’ of type CharObject was created
import datetime import time import re nodes = [ ('35.154.247.92', 10800), ('35.154.247.92', 10801), ] conn = psycopg2.connect( dbname="VTIONData", host="vtionproddb.chgz4nqwpdta.ap-south-1.rds.amazonaws.com", user="******", password="******") cur = conn.cursor() client = Client() client.connect(nodes) def filter(song_name): return (song_name.replace('_', ' ').replace(':', '').replace( '|', ' ').replace('-', ' ').replace('+', ' ').replace(' ', ' ').replace( '\"', '').replace('#', '').replace('(', '').replace(')', '').replace('[', '')) def remove_bdata(toreg): return re.sub("[\(\[].*?[\)\]]", "", toreg)
def artistgenre(): api_token = '905748e88e234954c9849597855d2d57' api_url_base = 'http://api.musixmatch.com/ws/1.1/track.search' headers = {'Content-Type': 'application/json','Authorization': 'Bearer {0}'.format(api_token)} # es = Elasticsearch([{'host': '13.235.1.36', 'port': 9200}]) # nodes = [ # ('127.0.0.1', 10800), # ('127.0.0.1', 10801), # ] nodes = [('35.154.247.92', 10800), ('35.154.247.92', 10801)] val = str(date.today()- timedelta(days=1)).split("-") datelast = val[1]+'/'+val[2]+'/'+val[0] # print(datelast) # datelast = '07/20/2019' # res = es.search(size=2000,index="tendays", body= {"query": { # "bool": { # "must": [ # {"match_phrase": { # "created_date.keyword": datelast # }}, # {"match_phrase": { # "events.key.keyword": "Audio_Tuned" # }} # ] # } # }}) # es = Elasticsearch([{'host': '13.235.1.36', 'port': 9200}]) #datelast = '09/18/2019' es = Elasticsearch([{'host': '13.235.1.36', 'port': 9200}]) body =\ { "query": { "bool": { "must": [ {"match_phrase": { "created_date.keyword": datelast }}, {"match_phrase": { "events.key.keyword": "Audio_Tuned" }} ] } } } res = es.search(index='tendays', body=body, size=10000) print(res) client = Client() client.connect(nodes) i = 0 for item in res["hits"]["hits"]: i = i + 1 try: d = removesmall(item["_source"]["events"]["segmentation"]["Song"]) d = removebig(d) print(d) QUERY = ''' SELECT ID FROM PUBLIC.SONG_DATA Where TRACKNAME = '{}'; '''.format(d) result = client.sql(QUERY) ignite = [] for trackid in result: ignite.append(trackid) if ignite: print("Song already present in 1st table :", d) print("Track ID ===========================:", ignite) pass else: # d = 'Touch the Floor (feat. Masego)' try: QUERY = '''SELECT ID FROM PUBLIC.SONG_DATA_ADD Where VSONG = '{}'; '''.format(d) print("CHECKING of second table query : ", QUERY) result = client.sql(QUERY) print("reached herer..............................") # print(next(result)) except Exception as e: print(e) print("outside the vla") ignite_add = [] for trackid in result: print("############################################") print(trackid[0]) ignite_add.append(trackid[0]) if ignite_add: print("Song 2nd table:", d) print("Track ID ======================= C:", ignite_add) pass else: print("api call") api_url = '{0}?q_track={1}&page_size=1&page=1&s_track_rating=desc&apikey=905748e88e234954c9849597855d2d57'.format(api_url_base,d) response = requests.get(api_url, headers=headers) print(response) if response.status_code == 200: track_data = str(response.content) print(track_data) track_data = track_data[:1] + '"' + track_data[2:] track_data = track_data[:-2] + '"' # print(track_data) track_id = json.loads(response.content.decode('utf-8'))["message"]["body"]["track_list"][0]["track"]["track_id"] track_name = json.loads(response.content.decode('utf-8'))["message"]["body"]["track_list"][0]["track"]["track_name"] track_name = track_name.replace("'","\\'") album_name = json.loads(response.content.decode('utf-8'))["message"]["body"]["track_list"][0]["track"]["album_name"] album_name = album_name.replace("'","\\'") artist_name = json.loads(response.content.decode('utf-8'))["message"]["body"]["track_list"][0]["track"]["artist_name"] artist_name = artist_name.replace("'","\\'") try: genre = json.loads(response.content.decode('utf-8'))["message"]["body"]["track_list"][0]["track"]["primary_genres"]["music_genre_list"][0]["music_genre"]["music_genre_name"] except Exception as e: genre = "-" QUERY = ''' SELECT MAX(ID) FROM PUBLIC.SONG_DATA; ''' re = client.sql( QUERY, include_field_names=True, ) print(next(re)) sqlId = next(re)[0] + 1 print("SQL ID ASSIGNED : ", sqlId) INS_QUERY = """INSERT INTO PUBLIC.SONG_DATA (ID, TRACKID, TRACKDATA, TRACKNAME, ALBUMNAME, ARTISTNAME, GENRE, STATUS) VALUES({sqlId}, {trackid}, '{track_data}', '{track_name}', '{album_name}', '{artist_name}', '{genre}', 0);""".format(sqlId=sqlId,track_name=d,artist_name=artist_name,album_name=album_name,genre=genre,trackid=track_id,track_data=track_data) print(INS_QUERY) try: client.sql(INS_QUERY) except Exception as e: print("Passed due to same data : ", e) print("===========================================================================") QUERY = ''' SELECT MAX(ID) FROM PUBLIC.SONG_DATA_ADD; ''' rre = client.sql( QUERY, include_field_names=True, ) print(next(rre)) try: sql = next(rre)[0] + 1 print("Assigned SQL ID : ",sql) except: sql = 1 print(" =========================> ", sql , type(sql)) INS = """INSERT INTO SONG_DATA_ADD (ID, TRACKID, TRACKDATA, VSONG, TRACKNAME, ALBUMNAME, ARTISTNAME, GENRE, STATUS) VALUES({sqlId}, {trackid}, '{track_data}','{vsong}', '{track_name}', '{album_name}', '{artist_name}', '{genre}', 0);""".format(sqlId=sql,vsong = d,track_name=track_name,artist_name=artist_name,album_name=album_name,genre=genre,trackid=track_id,track_data=track_data) print(INS) try: client.sql(INS) except Exception as e: print("EROOOOOOOR : ",e) pass str_id = meta + str(track_id) print("Yes error is there") print(str_id) doc = { "track_id":track_id, "track_name":track_name, "album_name":album_name, "artist_name":artist_name, "genre":genre } res = es.index(index="songs", doc_type='mixmatch', id=str_id ,body=doc) print(res['result']) else: print("no data") except Exception as e: # raise e pass print(i)
from pyignite import Client from cache.instance_data import PerperInstanceData from cache.stream_data import StreamData from model.state import State from model.context import Context from services.serializer import Serializer serializer = Serializer() ignite = Client() ignite.connect('localhost', 10800) ignite.register_binary_type(StreamData) instance = PerperInstanceData(ignite, serializer) state = State(instance, ignite, serializer) context = Context(instance, None, state, ignite) context.stream_action() print("Reading stream") print(ignite.get_cache_names()) streams_cache = ignite.get_cache("streams") for el in streams_cache.scan(): print(el)
# limitations under the License. from pyignite import Client from pyignite.datatypes.cache_config import CacheMode from pyignite.datatypes.prop_codes import * from pyignite.exceptions import SocketError nodes = [ ('127.0.0.1', 10800), ('127.0.0.1', 10801), ('127.0.0.1', 10802), ] client = Client(timeout=4.0) client.connect(nodes) print('Connected to {}'.format(client)) my_cache = client.get_or_create_cache({ PROP_NAME: 'my_cache', PROP_CACHE_MODE: CacheMode.REPLICATED, }) my_cache.put('test_key', 0) # abstract main loop while True: try: # do the work test_value = my_cache.get('test_key') my_cache.put('test_key', test_value + 1) except (OSError, SocketError) as e:
from collections import OrderedDict from pyignite import Client, GenericObjectMeta from pyignite.datatypes import * class Person(metaclass=GenericObjectMeta, schema=OrderedDict([ ('first_name', String), ('last_name', String), ('age', IntObject), ])): pass client = Client() client.connect('localhost', 10800) person_cache = client.get_or_create_cache('person') person_cache.put( 1, Person(first_name='Ivan', last_name='Ivanov', age=33) ) person = person_cache.get(1) print(person.__class__.__name__) # Person print(person.__class__ is Person) # True if `Person` was registered automatically (on writing) # or manually (using `client.register_binary_type()` method). # False otherwise
def export_users_csv(): print("Hello") nodes = [ ('35.154.247.92', 10800), ('35.154.247.92', 10801), ] conn = psycopg2.connect( dbname="VTIONData", host="vtionproddb.chgz4nqwpdta.ap-south-1.rds.amazonaws.com", user="******", password="******") cur = conn.cursor() client = Client() client.connect(nodes) yesterday = date.today() - timedelta(days=1) yesterdayMidnight = int(time.mktime(yesterday.timetuple())) - 19800 print("FROM : ", yesterdayMidnight) todayMidnight = yesterdayMidnight + 86400 print(todayMidnight) yesterdayMidnight = '1579199400' todayMidnight = '1579285800' QUERY = ''' SELECT ID, APPID, EVENTS, DEVICE_ID, "TIMESTAMP", MTIMESTAMP, UPTIMESTAMP, IPADDRESS, CITY, COUNTRY, EVENTNAME, CREATED_DATE, CREATED_TIME FROM PUBLIC.EVENTSDATA WHERE "TIMESTAMP" > '{}' AND "TIMESTAMP" < '{}' ;'''.format( yesterdayMidnight, todayMidnight) result = client.sql( QUERY, include_field_names=True, ) next(result) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) print(BASE_DIR) namespac = BASE_DIR + str("/target/Data_" + yesterday.strftime('%m_%d_%Y') + '.csv') with open(namespac, 'w+') as writeFile: print("START : ", time.localtime(time.time())) writer = csv.writer(writeFile) writer.writerow([ 'appid', 'DeviceId', 'vtionid', 'deviceModel', 'platform', 'apppackage', 'keyname', 'mobileOperator', 'app', 'song', 'album', 'Pstate', 'Program', 'Episode', 'Source', 'ipaddress', 'city', 'station', 'duration', 'timestamp', 'created_date', 'created_time', 'stationName', 'artist_name', 'genre', 'education', 'ownership', 'nccs_code', 'age', 'gender', 'number', 'Uninstall', 'Reward_option' ]) i = 0 for row in result: nccs = age = gender = number = status = '' artist_name = '' genre = '' try: cur.execute( '''SELECT nccs_code FROM public.installdata WHERE deviceid = '{}';''' .format(row[6])) nccs_result = cur.fetchone() nccs_res = nccs_result[0] except Exception as e: nccs_res = '' pass if i == 0: x = row[2] i = 1 else: x = row[5] urlSafeEncodedBytes = base64.urlsafe_b64encode( row[6].encode("utf-8")) vtionid = str(urlSafeEncodedBytes, "utf-8") try: y = json.loads(x) if y.get("segmentation") is None: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), "", "", "", "", "", "", "", row[10], row[11], "", "", row[7], row[3], row[4] ]) else: if y.get("ap") == "com.video.meter": song = y.get("segmentation").get("Song") app = y.get("segmentation").get("App") education = y.get("segmentation").get( "Highest Education") ownership = y.get("segmentation").get("Ownership") if row[2] == 'FM_Tuned': try: QUERYa = ''' SELECT STATION FROM MASTERTABLEDEMO WHERE CITY = '{}' AND FREQUENCY = '{}'; '''.format( row[11], y.get("segmentation").get("Station")) resulta = client.sql( QUERYa, include_field_names=True, ) next(resulta) stationName = [rowa for rowa in resulta] writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get("PState"), "", "", y.get("segmentation").get("Source"), row[10], row[11], y.get("segmentation").get("Station"), y.get("segmentation").get("Duration"), row[7], row[3], row[4], stationName[0][0], "", "", "", "", nccs_res ]) except Exception as e: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get("PState"), "", "", y.get("segmentation").get("Source"), row[10], row[11], y.get("segmentation").get("Station"), y.get("segmentation").get("Duration"), row[7], row[3], row[4], "", "", "", "", "", nccs_res ]) elif row[2] == 'Video_Tuned' or row[ 2] == 'Video_off': try: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get("PState"), y.get("segmentation").get("Program"), y.get("segmentation").get("Episode"), y.get("segmentation").get("Source"), row[10], row[11], "", y.get("segmentation").get("Duration"), row[7], row[3], row[4], "", "", "", "", "", nccs_res ]) except Exception as e: pass elif row[2] == 'Audio_Tuned': try: if song: if song.isdigit(): continue song = song.strip() filter_new = [ 'Download', 'download', 'AUD', 'Advertise', '%' ] status_filter = ( filter_new[0] in song or filter_new[1] in song or filter_new[2] in song or filter_new[3] in song or filter_new[4] in song) if str(status_filter) == 'False': album = y.get("segmentation").get( "Album") if album is None: album_filter = False else: a_filter = [ 'unknown', 'Advertise', 'Sponsored' ] album_filter = ( a_filter[0] in album or a_filter[1] in album or a_filter[2] in album) if str(album_filter) == 'False': QUERY = '''SELECT ARTISTNAME, GENRE FROM PUBLIC.SONG_DATA WHERE TRACKNAME = '{}';'''.format( song) details = client.sql( QUERY, include_field_names=True, ) next(details) for det in details: artist_name = det[0] genre = det[1] if artist_name: pass else: QUERY = '''SELECT ARTISTNAME, GENRE FROM PUBLIC.SONG_DATA_ADD WHERE TRACKNAME = '{}';'''.format( song) details = client.sql( QUERY, include_field_names= True, ) next(details) for det in details: artist_name = det[0] genre = det[1] if app == 'Amazon Music': pass if '%' in song or song.isdigit( ): continue else: if 'ñ' in song: song = song.replace( 'ñ', 'n') else: pass song = deEmojify( filter( remove_bdata(song). replace( ')', '').replace( '(', ''))) elif app == 'Spotify' or 'Spot' in app: if '%' in song: continue else: song = filter( deEmojify( remove_bdata( song))) elif app == 'Gaana': if 'Gaana' in song or song.isdigit( ) or song == ',': continue else: if 'ñ' in song or '├▒' in song: song = song.replace( 'ñ', 'n').replace( '├▒', 'n') else: pass song = select_before_list( remove_bdata( deEmojify(song)) ).lstrip('Ep 0123456789-.') elif app == 'Google Play Music': aa = song if song.isdigit( ) or 'high quality' in song or song == ',': continue else: song = domain( remove_bdata(song) ).lstrip( ' 0123456789.-' ).strip() if '128k' in song or '256k' in song or 'p3' in song: song = song.replace( '(128k)', '' ).replace( '(256k)', '').replace( '128k', '' ).replace( '256k', '' ).replace( '.mp3', '') song = select_before_list( filter( deEmojify( song))) elif app == 'Hungama': song = filter( remove_bdata( deEmojify(song))) elif app == 'i Music': song = song.replace( '-', ' ').replace( ':', ' ').replace( ' ', ' ').replace( ' ', ' ' ).replace( '.mp3', '') if 'Unknown artist' in song or 'i Music' in song or '//' in song: continue song = deEmojify( select_before_list( filter( domain( remove_bdata( song)) ).lstrip( ' 0123456789.-' ).strip())).strip( ) elif app == 'JioSaavn': song = filter( remove_bdata(song)) elif app == 'Saavn': #combine with jio savan song = filter( domain( remove_bdata( song))) app = 'JioSaavn' elif app == 'MX Player': if '--:--:--' in song or 'Insufficient balance' in song or 'Expiry Date' in song: continue else: song = deEmojify( filter( remove_bdata( song) ).replace( ' ', ' ')).strip() elif app == 'Music Player': song = domain( filter( remove_bdata(song)) ).replace( ' ', ' ').lstrip( ' 0123456789.-' ).strip() elif app == 'Wynk Music': song = remove_bdata( deEmojify(song) ).lstrip(' 0123456789.-' ).strip() if song.isdigit( ) or '%' in song: continue elif '(From' in song: song = song.replace( '(From', '') else: if '")' in song: song = remove_bdata( song.replace( '\")', '')) elif app == 'YouTube Music': song = filter( deEmojify( remove_bdata( song))) else: song = song app = app try: if song: if song.isdigit(): continue song = song.strip() writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network" ).get("ope"), app, song, y.get( "segmentation" ).get("Album"), y.get( "segmentation" ).get("PState"), "", "", y.get( "segmentation" ).get("Source"), row[10], row[11], y.get( "segmentation" ).get("Station"), y.get( "segmentation" ).get("Duration"), row[7], row[3], row[4], "", artist_name, genre, '', '', nccs_res, '', '', '', '' ]) else: pass except Exception as e: pass else: pass else: pass except Exception as e: pass elif row[2] == 'Register' or row[2] == 'Profile': if ownership: try: own = ownership.split(',') if len(own) >= 9: num_own = 9 else: num_own = len(own) cur.execute( '''SELECT nccs_code FROM public.nccs_flat where education = '{}' and ownership = '{}';'''. format(education, num_own)) nccs = cur.fetchone() except Exception as e: pass try: try: cur.execute( '''SELECT payment_option FROM public.payment_option where deviceid = '{}'; '''.format(row[6])) payment_option = cur.fetchone() payment = payment_option[0] except: payment = 'Amazon voucher' pass age = y.get("segmentation").get("age") if age: pass else: age = y.get("segmentation").get("Age") gender = y.get("segmentation").get( "Gender") if gender: pass else: gender = y.get("segmentation").get( "gender") number = y.get("segmentation").get( "Mobile Number") num_status = "+" in number if str(num_status) == 'True': number = '' else: try: cur.execute( '''SELECT i_status FROM public.appsflyer where number = '{}';'''.format( number)) status_now = cur.fetchone() status = status_now[0] if status != 'True': status = '' except: pass try: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get( "PState"), "", "", y.get("segmentation").get( "Source"), row[10], row[11], y.get("segmentation").get( "Station"), y.get("segmentation").get( "Duration"), row[7], row[3], row[4], "", artist_name, genre, education, ownership, nccs[0], age, gender, number, status, payment ]) except Exception as e: pass writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get( "PState"), "", "", y.get("segmentation").get( "Source"), row[10], row[11], y.get("segmentation").get( "Station"), y.get("segmentation").get( "Duration"), row[7], row[3], row[4], "", artist_name, genre, education, ownership, "", age, gender, number, status, payment ]) except Exception as e: pass else: writer.writerow([ row[1], row[6], vtionid, y.get("d"), y.get("p"), y.get("ap"), row[2], y.get("network").get("ope"), y.get("segmentation").get("App"), y.get("segmentation").get("Song"), y.get("segmentation").get("Album"), y.get("segmentation").get("PState"), "", "", y.get("segmentation").get("Source"), row[10], row[11], y.get("segmentation").get("Station"), y.get("segmentation").get("Duration"), row[7], row[3], row[4], "", artist_name, genre, education, ownership, "" ]) except Exception as e: pass client.close() writeFile.close()
# (the "License"); you may not use this file except in compliance with # the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from pyignite import Client from pyignite.datatypes import CharObject, ShortObject client = Client() client.connect('127.0.0.1', 10800) my_cache = client.get_or_create_cache('my cache') my_cache.put('my key', 42) # value ‘42’ takes 9 bytes of memory as a LongObject my_cache.put('my key', 42, value_hint=ShortObject) # value ‘42’ takes only 3 bytes as a ShortObject my_cache.put('a', 1) # ‘a’ is a key of type String my_cache.put('a', 2, key_hint=CharObject) # another key ‘a’ of type CharObject was created