def ensure_tables():
overrides['server'] = 'disco://localhost'
overrides['dump'] = False
overrides['nest'] = False
settings = Settings()
ddfs = settings['ddfs']
imps = Table.create(IMPS,
columns=['wide index string token', 'trie url', 'index trie site_id', 'uint cpm_millis',
'index int ad_id', 'index string date', 'index uint time', 'bit click',
'index bit impression', 'bit conversion'],
partition='date',
force=True)
pixels = Table.create(PIXELS,
columns=['wide index string token', 'index bit isActive', 'index trie site_id',
'uint amount', 'index int account_id', 'index trie city', 'index trie16 state',
'index int16 metro', 'string ip', 'lz4 keyword', 'index string date'],
partition='date',
force=True)
tags = ddfs.list("hustle:%s:" % IMPS)
if len(tags) == 0:
# insert the files
insert(imps, phile='fixtures/imps.json', preprocess=imp_process)
tags = ddfs.list("hustle:%s:" % PIXELS)
if len(tags) == 0:
# insert the files
insert(pixels, phile='fixtures/pixel.json')
def ensure_tables():
overrides['server'] = 'disco://localhost'
overrides['dump'] = False
overrides['nest'] = False
settings = Settings()
ddfs = settings['ddfs']
imps = Table.create(IMPS,
fields=[
'=$token', '%url', '+%site_id', '@cpm_millis',
'+#ad_id', '+$date', '+@time'
],
partition='date',
force=True)
pixels = Table.create(PIXELS,
fields=[
'=$token', '+@1isActive', '+%site_id', '@amount',
'+#account_id', '+%city', '+%2state', '+#2metro',
'$ip', '*keyword', '+$date'
],
partition='date',
force=True)
tags = ddfs.list("hustle:%s:" % IMPS)
if len(tags) == 0:
# insert the files
insert(imps, phile='fixtures/imps.json', preprocess=imp_process)
tags = ddfs.list("hustle:%s:" % PIXELS)
if len(tags) == 0:
# insert the files
insert(pixels, phile='fixtures/pixel.json')
def ensure_tables():
overrides['server'] = 'disco://localhost'
overrides['dump'] = False
overrides['nest'] = False
settings = Settings()
ddfs = settings['ddfs']
imps = Table.create(IMPS,
columns=[
'wide index string token', 'trie url',
'index trie site_id', 'uint cpm_millis',
'index int ad_id', 'index string date',
'index uint time', 'bit click',
'index bit impression', 'bit conversion'
],
partition='date',
force=True)
pixels = Table.create(PIXELS,
columns=[
'wide index string token', 'index bit isActive',
'index trie site_id', 'uint amount',
'index int account_id', 'index trie city',
'index trie16 state', 'index int16 metro',
'string ip', 'lz4 keyword', 'index string date'
],
partition='date',
force=True)
pixel_hlls = Table.create(PIXELS_HLL,
columns=[
'index bit isActive', 'index trie site_id',
'index int account_id', 'index trie city',
'index trie16 state', 'index string date',
'binary hll'
],
partition='date',
force=True)
ips = Table.create(IPS,
columns=['index trie16 exchange_id', 'index uint32 ip'],
force=True)
tags = ddfs.list("hustle:%s:" % IMPS)
if len(tags) == 0:
# insert the files
insert(imps, File='fixtures/imps.json', preprocess=imp_process)
tags = ddfs.list("hustle:%s:" % PIXELS)
if len(tags) == 0:
# insert the files
insert(pixels, File='fixtures/pixel.json')
tags = ddfs.list("hustle:%s:" % IPS)
if len(tags) == 0:
# insert the files
insert(ips, File='fixtures/ip.json')
tags = ddfs.list("hustle:%s:" % PIXELS_HLL)
if len(tags) == 0:
# insert the files
insert_hll(pixel_hlls, file='./fixtures/pixel.json', hll_field='token')
def ensure_tables():
overrides['server'] = 'disco://localhost'
overrides['dump'] = False
overrides['nest'] = False
settings = Settings()
ddfs = settings['ddfs']
pcappacket= Table.create(mmm,columns=['index int32 GMTtime','index int32 MicroTime','index int32 caplen','index int32 len','index string ethtype','index string protocol','index uint8 ttl','index uint8 sip0','index uint8 sip1','index uint8 sip2','index uint8 sip3','index uint8 dip0','index uint8 dip1','index uint8 dip2','index uint8 dip3','index uint16 sport','index uint16 dport','lz4 data'],partition=None,force=True)
'''
imps = Table.create(IMPS,
columns=['wide index string token', 'trie url', 'index trie site_id', 'uint cpm_millis',
'index int ad_id', 'index string date', 'index uint time'],
partition='date',
force=True)
pixels = Table.create(PIXELS,
columns=['wide index string token', 'index uint8 isActive', 'index trie site_id',
'uint amount', 'index int account_id', 'index trie city', 'index trie16 state',
'index int16 metro', 'string ip', 'lz4 keyword', 'index string date'],
partition='date',
force=True)
'''
from ls import *
ls('/home/xiner/hustle/pcapread/testall.pcap')
tags = ddfs.list("hustle:%s:" % mmm)
print len(tags)
if len(tags) == 0:
# insert the files
insert(pcappacket, phile='result.json' )
'''tags = ddfs.list("hustle:%s:" % PIXELS)
def ensure_tables():
overrides["server"] = "disco://localhost"
overrides["dump"] = False
overrides["nest"] = False
settings = Settings()
ddfs = settings["ddfs"]
imps = Table.create(
IMPS,
fields=["=$token", "%url", "+%site_id", "@cpm_millis", "+#ad_id", "+$date", "+@time"],
partition="date",
force=True,
)
pixels = Table.create(
PIXELS,
fields=[
"=$token",
"+@1isActive",
"+%site_id",
"@amount",
"+#account_id",
"+%city",
"+%2state",
"+#2metro",
"$ip",
"*keyword",
"+$date",
],
partition="date",
force=True,
)
tags = ddfs.list("hustle:%s:" % IMPS)
if len(tags) == 0:
# insert the files
insert(imps, phile="fixtures/imps.json", preprocess=imp_process)
tags = ddfs.list("hustle:%s:" % PIXELS)
if len(tags) == 0:
# insert the files
insert(pixels, phile="fixtures/pixel.json")
def ensure_tables():
overrides['server'] = 'disco://localhost'
overrides['dump'] = False
overrides['nest'] = False
settings = Settings()
ddfs = settings['ddfs']
imps = Table.create(IMPS,
fields=['=$token', '%url', '+%site_id', '@cpm_millis', '+#ad_id', '+$date', '+@time'],
partition='date',
force=True)
tags = ddfs.list("hustle:%s:" % IMPS)
if len(tags) == 0:
# insert the files
insert(imps, File='fixtures/imps.json', preprocess=imp_process)
return imps