def minertables(self):
resources = base.get_resources()
existing = resources.keys()
package_id = base.get_package_id()
new_fields = []
tables = self.config.get('settings','tables').split(',')
non_user_tables = self.config.get('settings','non_user_tables').split(',')
for table in tables + non_user_tables:
if table in existing:
continue
if table not in non_user_tables:
fieldNames = ['uid'] + self.config.get(table,'fields').split(',')
fieldTypes = ['bigint'] + self.config.get(table,'field_types').split(',')
else:
fieldNames = self.config.get(table,'fields').split(',')
fieldTypes = self.config.get(table,'field_types').split(',')
fields = [ {'id':field[0], 'type':field[1]} for field in zip(fieldNames,fieldTypes) ]
print "Creating table: "+table
new_fields.append(self.local.action.datastore_create(resource={'package_id':package_id, 'name':table }, fields=fields))
tables = existing + [ table['resource']['name'] for table in new_fields ]
res_ids = [ resources[key] for key in existing ] + [ table['resource_id'] for table in new_fields ]
self.push_settings('tables',','.join(tables))
self.push_settings('resources',','.join(res_ids))
self.local.action.resource_view_create(resource_id=resources[table],title='default view',view_type='recline_view')
print 'done'
def push(self):
if len(self.args) >= 3:
fname = self.args[1]
table = self.args[2]
else:
if len(self.args) == 2:
fname = self.args[1]
table = fname.split('/')[-1].split('.')[0]
else:
print "USAGE: push <csvfile> <table>"
return
resources = base.get_resources()
res = resources.get(table,False)
if not res:
if table in resources.values():
res = table
else:
print "No such table: "+table
return
non_user_tables = self.config.get('settings','non_user_tables').split(',')
if table not in non_user_tables:
fields = ['uid'] + self.config.get(table,'fields').split(',')
else:
fields = self.config.get(table,'fields').split(',')
try:
infile = open(fname,'rb')
except:
print "Can't open "+fname
clean_text = lambda t: filter(lambda c: c == '\n' or 32 <= ord(c) <= 126,t)
field_types = self.config.get(table,'field_types').split(',')
is_list = [ ft[-2:] == '[]' for ft in field_types ]
if True in is_list:
row_getter = lambda r: [ [ clean_text(i) for i in ast.literal_eval(e[1]) ] if is_list[e[0]] else clean_text(e[1])
for e in enumerate(r[1:]) ]
else:
row_getter = lambda r: r[1:]
reader = csv.reader(infile, delimiter=',')
reader.next()
data = []
for row in reader:
data.append(dict(zip(fields,row_getter(row))))
if len(data) == 128:
self.local.action.datastore_upsert(resource_id=res,records=data,method='insert')
#print '.',
data = []
if len(data):
self.local.action.datastore_upsert(resource_id=res,records=data,method='insert')
def drop_table(self):
tables = self.args[1]
resources = base.get_resources()
if tables == 'all':
kill = resources.values()
else:
existing = resources.keys()
kill = [ resources.get(tab) for tab in tables.split(',') if tab in existing ]
for tab in kill:
self.local.action.datastore_delete(resource_id=resources[tab])
def minertables(self):
existing = base.get_resources().keys()
package_id = base.get_package_id()
tables = self.config.get('settings','tables').split(',')
non_user_tables = self.config.get('settings','non_user_tables').split(',')
for table in tables + non_user_tables:
if table not in non_user_tables:
fieldNames = ['uid'] + self.config.get(table,'fields').split(',')
fieldTypes = ['bigint'] + self.config.get(table,'field_types').split(',')
else:
fieldNames = self.config.get(table,'fields').split(',')
fieldTypes = self.config.get(table,'field_types').split(',')
fields = [ {'id':field[0], 'type':field[1]} for field in zip(fieldNames,fieldTypes) ]
if table not in existing:
print "Creating table: "+table
self.local.action.datastore_create(resource={'package_id':package_id, 'name':table }, fields=fields)
def push(self):
if len(self.args) >= 3:
table = self.args[1]
fname = self.args[2]
else:
print "USAGE: push <table> <csvfile>"
return
res = base.get_resources().get(table,False)
if not res:
print "No such table: "+table
return
non_user_tables = self.config.get('settings','non_user_tables').split(',')
if table not in non_user_tables:
fields = ['uid'] + self.config.get(table,'fields').split(',')
else:
fields = self.config.get(table,'fields').split(',')
try:
infile = open(fname,'rb')
except:
print "Cant't open "+fname
reader = csv.reader(infile, delimiter=',')
reader.next()
data = []
for row in reader:
data.append(dict(zip(fields,row[1:])))
if len(data) == 50:
self.local.action.datastore_upsert(resource_id=res,records=data,method='insert')
print '.',
data = []
if len(data):
self.local.action.datastore_upsert(resource_id=res,records=data,method='insert')
# Licensed under the Apache License Version 2.0: http://www.apache.org/licenses/LICENSE-2.0.txt
__author__ = 'Giles Richard Greenway'
from itertools import chain
import base
import dateutil.parser
get_weekday = lambda date: base.weekdays[dateutil.parser.parse(date).isoweekday()]
resources = base.get_resources()
local = base.get_local()
quotify = lambda x: '"'+x+'"'
get_weekday = lambda date: base.weekdays[dateutil.parser.parse(date).isoweekday()]
table_field_types = base.get_field_types()
def quotify_field_value(table,field,value):
if table_field_types[table].get(field,'bigint') == 'text':
return "'"+str(value)+"'"
else:
return str(value)
def value_comp(table,field,value,op='='):
return op.join([field,quotify_field_value(table,field,value)])
def select(fields,table,eq={},gt={},lt={},ge={},ne={},where=False,group=False,having=False,order=False,page=False,page_size=128,distinct=True):
clauses = []
where_clause = group_clause = having_clause = order_clause = page_clause = False
def push_cells(self):
try:
cell_file = open(self.args[1],'rb')
except:
return
resources = base.get_resources()
timestamp = datetime.now().isoformat()
all_the_mcc = [ record['mcc'] for record in self.local.action.datastore_search_sql(sql=select(['mcc'],'gsmcell',ne={'mcc':'None'}))['records'] ]
mnc_map = dict([ (mcc,all_the_mnc(mcc)) for mcc in all_the_mcc ])
lac_map = {}
#print all_the_mcc
#print all_the_mnc
tick = 0
found = 0
sfound = '0'
last_mcc = 0
last_mnc = 0
smcc = '0'
smnc = '0'
for line in cell_file.readlines():
if line.split(',')[0] == 'radio':
idx = {'mcc':1, 'mnc':2, 'lac':3, 'cid':4, 'lon':6, 'lat':7, 'changeable':10 }
change = lambda x: x
else:
idx = {'mcc':0, 'mnc':1, 'lac':2, 'cid':3, 'lon':4, 'lat':5, 'changeable':7 }
change = lambda x: 1 if x else 0
break
cell_file.close()
cell_file = open(self.args[1],'rb')
for line in cell_file.readlines():
if tick % 256 == 0:
print "Searching. MCC:" + smcc + " MNC:" + smnc + " Found: "+sfound
tick = 1
else:
tick += 1
chunks = line.split(',')
#print chunks
if len(chunks) < 11:
continue
mcc = chunks[idx['mcc']]
mnc = chunks[idx['mnc']]
if last_mcc <> mcc:
smcc = str(mcc)
if last_mnc <> mnc:
smnc = str(mnc)
if mcc in all_the_mcc:
if mnc in mnc_map[mcc]:
lac_key = smcc+'_'+smnc
lac = chunks[idx['lac']]
if not lac_map.get(lac_key,False):
sql_query = select(['lac'],'gsmcell',eq={'mcc':smcc,'mnc':smnc})
lac_map[lac_key] = dict([ (r['lac'],True) for r in self.local.action.datastore_search_sql(sql=sql_query)['records'] ])
if lac_map[lac_key].get(lac,False):
cid = chunks[idx['cid']]
ex_eq = {'mcc':mcc,'mnc':mnc,'lac':lac,'cid':cid}
ex_query = 'cid NOT IN (' + select(['cid::text'],'gsmlocation',eq=ex_eq) + ')'
eq = {'mcc':smcc,'mnc':smnc,'lac':str(lac),'cid':str(cid)}
sql_query = select(['cid'],'gsmcell',eq=eq,where=[ex_query])
#sql_query = select(['cid'],'gsmcell',eq=eq)
#try:
insert = len(self.local.action.datastore_search_sql(sql=sql_query)['records']) > 0
#except:
# sql_query = select(['cid'],'gsmcell',eq=eq)
# insert = len(self.local.action.datastore_search_sql(sql=sql_query)['records']) > 0
if insert:
found += 1
sfound = str(found)
rendered_cell = {'mcc':mcc, 'mnc':mnc, 'lac':lac, 'cid':cid, 'lat':chunks[idx['lat']], 'lon':chunks[idx['lon']],
'changeable':change(chunks[idx['changeable']]), 'retrieved':timestamp}
print "Found: "+','.join([ rendered_cell[f] for f in ['mcc','mnc','lac','cid'] ])
self.local.action.datastore_upsert(resource_id=resources['gsmlocation'],records=[rendered_cell],method='insert')
last_mcc = mcc
last_mnc = mnc
def flush(self):
table = self.args[1]
if table in ['user','socket','gsmcell','mobilenetwork','wifinetwork','minerlog','notification','networktraffic']:
return
resources = base.get_resources()
self.local.action.datastore_delete(resource_id=resources[table])
def create_views(self):
resources = base.get_resources()
for table in resources.keys():
self.local.action.resource_view_create(resource_id=resources[table],title='default view',view_type='recline_view')
