def minertables(self): resources = base.get_resources() existing = resources.keys() package_id = base.get_package_id() new_fields = [] tables = self.config.get('settings','tables').split(',') non_user_tables = self.config.get('settings','non_user_tables').split(',') for table in tables + non_user_tables: if table in existing: continue if table not in non_user_tables: fieldNames = ['uid'] + self.config.get(table,'fields').split(',') fieldTypes = ['bigint'] + self.config.get(table,'field_types').split(',') else: fieldNames = self.config.get(table,'fields').split(',') fieldTypes = self.config.get(table,'field_types').split(',') fields = [ {'id':field[0], 'type':field[1]} for field in zip(fieldNames,fieldTypes) ] print "Creating table: "+table new_fields.append(self.local.action.datastore_create(resource={'package_id':package_id, 'name':table }, fields=fields)) tables = existing + [ table['resource']['name'] for table in new_fields ] res_ids = [ resources[key] for key in existing ] + [ table['resource_id'] for table in new_fields ] self.push_settings('tables',','.join(tables)) self.push_settings('resources',','.join(res_ids)) self.local.action.resource_view_create(resource_id=resources[table],title='default view',view_type='recline_view') print 'done'
def push(self): if len(self.args) >= 3: fname = self.args[1] table = self.args[2] else: if len(self.args) == 2: fname = self.args[1] table = fname.split('/')[-1].split('.')[0] else: print "USAGE: push <csvfile> <table>" return resources = base.get_resources() res = resources.get(table,False) if not res: if table in resources.values(): res = table else: print "No such table: "+table return non_user_tables = self.config.get('settings','non_user_tables').split(',') if table not in non_user_tables: fields = ['uid'] + self.config.get(table,'fields').split(',') else: fields = self.config.get(table,'fields').split(',') try: infile = open(fname,'rb') except: print "Can't open "+fname clean_text = lambda t: filter(lambda c: c == '\n' or 32 <= ord(c) <= 126,t) field_types = self.config.get(table,'field_types').split(',') is_list = [ ft[-2:] == '[]' for ft in field_types ] if True in is_list: row_getter = lambda r: [ [ clean_text(i) for i in ast.literal_eval(e[1]) ] if is_list[e[0]] else clean_text(e[1]) for e in enumerate(r[1:]) ] else: row_getter = lambda r: r[1:] reader = csv.reader(infile, delimiter=',') reader.next() data = [] for row in reader: data.append(dict(zip(fields,row_getter(row)))) if len(data) == 128: self.local.action.datastore_upsert(resource_id=res,records=data,method='insert') #print '.', data = [] if len(data): self.local.action.datastore_upsert(resource_id=res,records=data,method='insert')
def drop_table(self): tables = self.args[1] resources = base.get_resources() if tables == 'all': kill = resources.values() else: existing = resources.keys() kill = [ resources.get(tab) for tab in tables.split(',') if tab in existing ] for tab in kill: self.local.action.datastore_delete(resource_id=resources[tab])
def minertables(self): existing = base.get_resources().keys() package_id = base.get_package_id() tables = self.config.get('settings','tables').split(',') non_user_tables = self.config.get('settings','non_user_tables').split(',') for table in tables + non_user_tables: if table not in non_user_tables: fieldNames = ['uid'] + self.config.get(table,'fields').split(',') fieldTypes = ['bigint'] + self.config.get(table,'field_types').split(',') else: fieldNames = self.config.get(table,'fields').split(',') fieldTypes = self.config.get(table,'field_types').split(',') fields = [ {'id':field[0], 'type':field[1]} for field in zip(fieldNames,fieldTypes) ] if table not in existing: print "Creating table: "+table self.local.action.datastore_create(resource={'package_id':package_id, 'name':table }, fields=fields)
def push(self): if len(self.args) >= 3: table = self.args[1] fname = self.args[2] else: print "USAGE: push <table> <csvfile>" return res = base.get_resources().get(table,False) if not res: print "No such table: "+table return non_user_tables = self.config.get('settings','non_user_tables').split(',') if table not in non_user_tables: fields = ['uid'] + self.config.get(table,'fields').split(',') else: fields = self.config.get(table,'fields').split(',') try: infile = open(fname,'rb') except: print "Cant't open "+fname reader = csv.reader(infile, delimiter=',') reader.next() data = [] for row in reader: data.append(dict(zip(fields,row[1:]))) if len(data) == 50: self.local.action.datastore_upsert(resource_id=res,records=data,method='insert') print '.', data = [] if len(data): self.local.action.datastore_upsert(resource_id=res,records=data,method='insert')
# Licensed under the Apache License Version 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt __author__ = 'Giles Richard Greenway' from itertools import chain import base import dateutil.parser get_weekday = lambda date: base.weekdays[dateutil.parser.parse(date).isoweekday()] resources = base.get_resources() local = base.get_local() quotify = lambda x: '"'+x+'"' get_weekday = lambda date: base.weekdays[dateutil.parser.parse(date).isoweekday()] table_field_types = base.get_field_types() def quotify_field_value(table,field,value): if table_field_types[table].get(field,'bigint') == 'text': return "'"+str(value)+"'" else: return str(value) def value_comp(table,field,value,op='='): return op.join([field,quotify_field_value(table,field,value)]) def select(fields,table,eq={},gt={},lt={},ge={},ne={},where=False,group=False,having=False,order=False,page=False,page_size=128,distinct=True): clauses = [] where_clause = group_clause = having_clause = order_clause = page_clause = False
def push_cells(self): try: cell_file = open(self.args[1],'rb') except: return resources = base.get_resources() timestamp = datetime.now().isoformat() all_the_mcc = [ record['mcc'] for record in self.local.action.datastore_search_sql(sql=select(['mcc'],'gsmcell',ne={'mcc':'None'}))['records'] ] mnc_map = dict([ (mcc,all_the_mnc(mcc)) for mcc in all_the_mcc ]) lac_map = {} #print all_the_mcc #print all_the_mnc tick = 0 found = 0 sfound = '0' last_mcc = 0 last_mnc = 0 smcc = '0' smnc = '0' for line in cell_file.readlines(): if line.split(',')[0] == 'radio': idx = {'mcc':1, 'mnc':2, 'lac':3, 'cid':4, 'lon':6, 'lat':7, 'changeable':10 } change = lambda x: x else: idx = {'mcc':0, 'mnc':1, 'lac':2, 'cid':3, 'lon':4, 'lat':5, 'changeable':7 } change = lambda x: 1 if x else 0 break cell_file.close() cell_file = open(self.args[1],'rb') for line in cell_file.readlines(): if tick % 256 == 0: print "Searching. MCC:" + smcc + " MNC:" + smnc + " Found: "+sfound tick = 1 else: tick += 1 chunks = line.split(',') #print chunks if len(chunks) < 11: continue mcc = chunks[idx['mcc']] mnc = chunks[idx['mnc']] if last_mcc <> mcc: smcc = str(mcc) if last_mnc <> mnc: smnc = str(mnc) if mcc in all_the_mcc: if mnc in mnc_map[mcc]: lac_key = smcc+'_'+smnc lac = chunks[idx['lac']] if not lac_map.get(lac_key,False): sql_query = select(['lac'],'gsmcell',eq={'mcc':smcc,'mnc':smnc}) lac_map[lac_key] = dict([ (r['lac'],True) for r in self.local.action.datastore_search_sql(sql=sql_query)['records'] ]) if lac_map[lac_key].get(lac,False): cid = chunks[idx['cid']] ex_eq = {'mcc':mcc,'mnc':mnc,'lac':lac,'cid':cid} ex_query = 'cid NOT IN (' + select(['cid::text'],'gsmlocation',eq=ex_eq) + ')' eq = {'mcc':smcc,'mnc':smnc,'lac':str(lac),'cid':str(cid)} sql_query = select(['cid'],'gsmcell',eq=eq,where=[ex_query]) #sql_query = select(['cid'],'gsmcell',eq=eq) #try: insert = len(self.local.action.datastore_search_sql(sql=sql_query)['records']) > 0 #except: # sql_query = select(['cid'],'gsmcell',eq=eq) # insert = len(self.local.action.datastore_search_sql(sql=sql_query)['records']) > 0 if insert: found += 1 sfound = str(found) rendered_cell = {'mcc':mcc, 'mnc':mnc, 'lac':lac, 'cid':cid, 'lat':chunks[idx['lat']], 'lon':chunks[idx['lon']], 'changeable':change(chunks[idx['changeable']]), 'retrieved':timestamp} print "Found: "+','.join([ rendered_cell[f] for f in ['mcc','mnc','lac','cid'] ]) self.local.action.datastore_upsert(resource_id=resources['gsmlocation'],records=[rendered_cell],method='insert') last_mcc = mcc last_mnc = mnc
def flush(self): table = self.args[1] if table in ['user','socket','gsmcell','mobilenetwork','wifinetwork','minerlog','notification','networktraffic']: return resources = base.get_resources() self.local.action.datastore_delete(resource_id=resources[table])
def create_views(self): resources = base.get_resources() for table in resources.keys(): self.local.action.resource_view_create(resource_id=resources[table],title='default view',view_type='recline_view')