def import_nav_dat(ntype=None):
"""Imports the nav_dat"""
marker_count = 1000
c = 0
xrec = conf.get_current()
xdate = xrec['date']
shard_dir = conf.work_dir("/raw_data/nav")
if ntype != None:
if ntype == "vor":
file_path = "/home/fgxx/fgx-navdata/temp/nav.3.dat"
if ntype == "markers":
for marker_code in [RC.im, RC.mm, RC.om]:
file_path = "%s/%s.%s.dat" % (shard_dir, marker_code, xdate)
process_file(file_path, xrec)
process_file(file_path, xrec)
return
nuke_data()
for i in RC.list():
nt = RC.ntype(i)
file_path = "%s/%s.%s.dat" % (shard_dir, nt, xdate)
print file_path
process_file(file_path, xrec)
def shard(dir_path=None, src_name=None):
"""Import file 'apt.dat."""
nuke_data()
xrec = conf.get_current()
dir_path = conf.work_dir("/xplane_unzipped/%s" % xrec['zip_dir'])
reader = open(dir_path + "/apt.dat", 'r')
reader.next()
reader.next()
reader.next()
reader.next()
print "First 4 lines of apt.dat skipped."
xindex = []
apt_ident = ""
xlines = []
c = 4
for line in reader:
#print "=", c, line[0:10]
## strip line
lin = line.strip()
if len(lin) > 0:
## Its not a blank line so append
xlines.append(lin)
else:
apt_ident, ok = write_data_shard(xlines, xrec)
# print apt_ident
if ok:
xindex.append(apt_ident)
##file_path = ws_path = ""
##fn = open() ## Its a blank line so process xlines
#apt, err = process_apt(xlines, src_name)
#print "process=", apt, err
#if err == QUIRK:
# pass
#else:
# if apt != None:
# update_apt(apt)
xlines = []
if c % 1000 == 0:
print c, apt_ident if ok else "%s - skip" % apt_ident
c += 1
file_path = conf.work_dir( "/raw_data/apt/index.%s.json" % xrec['date'] )
conf.write_json(file_path, dict(airports=sorted(xindex), count=len(xindex)) )
def shard(dir_path=None, src_name=None):
"""Import file 'apt.dat."""
nuke_data()
xrec = conf.get_current()
dir_path = conf.work_dir("/xplane_unzipped/%s" % xrec['zip_dir'])
reader = open(dir_path + "/apt.dat", 'r')
reader.next()
reader.next()
reader.next()
reader.next()
print "First 4 lines of apt.dat skipped."
xindex = []
apt_ident = ""
xlines = []
c = 4
for line in reader:
#print "=", c, line[0:10]
## strip line
lin = line.strip()
if len(lin) > 0:
## Its not a blank line so append
xlines.append(lin)
else:
apt_ident, ok = write_data_shard(xlines, xrec)
# print apt_ident
if ok:
xindex.append(apt_ident)
##file_path = ws_path = ""
##fn = open() ## Its a blank line so process xlines
#apt, err = process_apt(xlines, src_name)
#print "process=", apt, err
#if err == QUIRK:
# pass
#else:
# if apt != None:
# update_apt(apt)
xlines = []
if c % 1000 == 0:
print c, apt_ident if ok else "%s - skip" % apt_ident
c += 1
file_path = conf.work_dir("/raw_data/apt/index.%s.json" % xrec['date'])
conf.write_json(file_path, dict(airports=sorted(xindex),
count=len(xindex)))
def import_fix_dat(xrec=None):
"""Reads earth_fix.dat and inserts to `fix` table
TODO:
fix the NPOLE bug
"""
xrec = conf.get_current()
regEx = re.compile("[A-Z]{5}")
inputfile = conf.work_dir("/xplane_unzipped/%s/earth_fix.dat" % (xrec['zip_dir']))
c = 0
with open(inputfile) as readnav:
for line in readnav:
c += 1
# Skip first three lines, hope Robin Peel will never change this behaviour ;-)
if c < 4:
pass
else:
if not line.startswith("99"):
lst = line.strip().split()
fix_ident = str(lst[2])
if fix_ident == "NPOLE":
pass
else:
#//fix_center_lat84 = str(lst[0])
#fix_center_lon84 = str(lst[1])
maj = 1 if regEx.match(fix_ident) else 0
# insert to db
insert_fix(
**dict(
ident=fix_ident, major=maj,
lat=str(lst[0]), lon=str(lst[1])
)
)
## We commit every x to make faster
if c % 5000 == 0:
print " > fix: %s - %s of %s" % (fix_ident, c, MAX_LINES_GUESS)
idx_file = conf.raw_fix_path()
def import_airports(apt_ident=None):
nuke_data()
xrec = conf.get_current()
xdate = xrec['date']
if apt_ident:
import_airport(apt_ident, xrec)
return
idx_file = conf.work_dir("/raw_data/apt/index.%s.json" % xrec['date'])
data = conf.read_json(idx_file)
print "processing - %s" % data['count']
for c, apt_ident in enumerate(data['airports']):
import_airport(apt_ident, xrec)
if c % 100 == 0:
print "%s of %s\t%s" % (c, data['count'], apt_ident)
def import_airports(apt_ident=None):
nuke_data()
xrec = conf.get_current()
xdate = xrec['date']
if apt_ident:
import_airport(apt_ident, xrec)
return
idx_file = conf.work_dir("/raw_data/apt/index.%s.json" % xrec['date'])
data = conf.read_json(idx_file)
print "processing - %s" % data['count']
for c, apt_ident in enumerate(data['airports']):
import_airport(apt_ident, xrec)
if c % 100 == 0:
print "%s of %s\t%s" % (c, data['count'], apt_ident)
def shard(xrec=None):
"""Reads earth_fix.dat and inserts to `fix` table
TODO:
fix the NPOLE bug
"""
xrec = conf.get_current()
print "FIX_DAT: shard()", xrec
zip_ver = xrec['zip_dir']
xindex = []
regEx = re.compile("[A-Z]{5}")
inputfile = conf.work_dir("/xplane_unzipped/%s/earth_fix.dat" % (xrec['zip_dir']))
c = 0
print inputfile
with open(inputfile) as readnav:
for line in readnav:
c += 1
# Skip first three lines, hope Robin Peel will never change this behaviour ;-)
if c < 4:
pass
else:
if not line.startswith("99"):
lst = line.strip().split()
fix_ident = str(lst[2])
fixblob = None
if fix_ident == "NPOLE":
pass
else:
## Write shard
blob_path = conf.raw_fix_path( xrec, fix_ident)
#print file_path, xrec
f = open(blob_path + ".txt", "w")
f.write(line)
f.close()
## make dic
maj = True if regEx.match(fix_ident) else False
data = dict(
ident=fix_ident, major=maj, src=line,
lat=str(lst[0]), lon=str(lst[1])
)
json_path = blob_path + ".json"
conf.write_json(json_path, data)
xindex.append(fix_ident)
if c % 5000 == 0:
print " > fix: %s - %s of %s" % (fix_ident, c, MAX_LINES_GUESS)