def test_local_export(self):
session = self.db_master_session
k = dict(mcc=1, mnc=2, lac=4)
gsm = RADIO_TYPE['gsm']
for i in range(190, 200):
session.add(Cell(radio=gsm, cid=i, lat=1.0, lon=2.0, **k))
# add one incomplete / unprocessed cell
session.add(Cell(cid=210, lat=None, lon=None, **k))
session.commit()
with selfdestruct_tempdir() as d:
path = os.path.join(d, 'export.csv.gz')
cond = and_(cell_table.c.cid != CELLID_LAC,
cell_table.c.lat.isnot(None))
write_stations_to_csv(session, cell_table, CELL_COLUMNS, cond,
path, make_cell_export_dict, CELL_FIELDS)
with GzipFile(path, "rb") as f:
r = csv.DictReader(f, CELL_FIELDS)
header = r.next()
self.assertTrue('area' in header.values())
self.assertEqual(header, CELL_HEADER_DICT)
cid = 190
for d in r:
t = dict(radio='GSM', cid=cid, **k)
t = dict([(n, str(v)) for (n, v) in t.items()])
self.assertDictContainsSubset(t, d)
cid += 1
self.assertEqual(r.line_num, 11)
self.assertEqual(cid, 200)
def test_local_export(self):
session = self.db_master_session
k = dict(mcc=1, mnc=2, lac=4, lat=1.0, lon=2.0)
for i in range(190, 200):
session.add(Cell(radio=RADIO_TYPE['gsm'], cid=i, **k))
session.commit()
with selfdestruct_tempdir() as d:
path = os.path.join(d, 'export.csv.gz')
write_stations_to_csv(session, cell_table, CELL_COLUMNS,
cell_table.c.cid != CELLID_LAC,
path, make_cell_export_dict, CELL_FIELDS)
with GzipFile(path, "rb") as f:
r = csv.DictReader(f, CELL_FIELDS)
header = r.next()
self.assertTrue('area' in header.values())
self.assertEqual(header, CELL_HEADER_DICT)
cid = 190
for d in r:
t = dict(radio='GSM', cid=cid, **k)
t = dict([(n, str(v)) for (n, v) in t.items()])
self.assertDictContainsSubset(t, d)
cid += 1
self.assertEqual(r.line_num, 11)
self.assertEqual(cid, 200)
def get_test_csv(self, lo=1, hi=10, time=1408604686):
line_template = ('GSM,{mcc},{mnc},{lac},{cid},{psc},{lon},'
'{lat},1,1,1,{time},{time},')
lines = [line_template.format(
cid=i * 1010, psc='',
lon=PARIS_LON + i * 0.002,
lat=PARIS_LAT + i * 0.001,
time=time,
**self.KEY)
for i in range(lo, hi)]
# add bad lines
lines.append(line_template.format(
mcc=FRANCE_MCC, mnc=VIVENDI_MNC,
lac='', cid='', psc=12,
lon=PARIS_LON, lat=PARIS_LAT, time=time,
))
lines.append(line_template.format(
mcc=FRANCE_MCC, mnc=VIVENDI_MNC,
lac='', cid='', psc='',
lon=PARIS_LON, lat=PARIS_LAT, time=time,
))
txt = '\n'.join(lines)
with selfdestruct_tempdir() as d:
path = os.path.join(d, 'import.csv.gz')
with GzipFile(path, 'wb') as f:
f.write(txt)
yield path
def get_test_csv(self, lo=1, hi=10, time=1408604686):
cell = self.cell
line_template = ('GSM,{mcc},{mnc},{lac},{cid},{psc},{lon},'
'{lat},1,1,1,{time},{time},')
lines = [line_template.format(
mcc=cell.mcc, mnc=cell.mnc, lac=cell.lac, cid=i * 1010, psc='',
lon=cell.lon + i * 0.002,
lat=cell.lat + i * 0.001,
time=time)
for i in range(lo, hi)]
# add bad lines
lines.append(line_template.format(
mcc=cell.mcc, mnc=cell.mnc,
lac='', cid='', psc=12,
lon=cell.lon, lat=cell.lat, time=time,
))
lines.append(line_template.format(
mcc=cell.mcc, mnc=cell.mnc,
lac='', cid='', psc='',
lon=cell.lon, lat=cell.lat, time=time,
))
txt = '\n'.join(lines)
with selfdestruct_tempdir() as d:
path = os.path.join(d, 'import.csv.gz')
with util.gzip_open(path, 'w') as gzip_file:
gzip_file.write(txt)
yield path
def test_local_export(self):
cell_fixture_fields = (
'radio', 'cid', 'lat', 'lon', 'mnc', 'mcc', 'lac')
base_cell = CellFactory.build(radio=Radio.gsm)
cell_key = {'radio': Radio.gsm, 'mcc': base_cell.mcc,
'mnc': base_cell.mnc, 'lac': base_cell.lac}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid, lat=base_cell.lat,
lon=base_cell.lon, **cell_key)
CellFactory(**cell)
cell['radio'] = 'GSM'
cell_strings = [
(field, str(value)) for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
CellFactory(cid=210, lat=None, lon=None, **cell_key)
self.session.commit()
with selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, 'export.csv.gz')
cond = Cell.__table__.c.lat.isnot(None)
write_stations_to_csv(
self.session, Cell.__table__, CELL_COLUMNS, cond,
path, make_cell_export_dict, CELL_FIELDS)
with util.gzip_open(path, 'r') as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = six.next(reader)
self.assertTrue('area' in header.values())
self.assertEqual(header, CELL_HEADER_DICT)
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value) for (field, value)
in exported_cell.items()
if field in cell_fixture_fields]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
self.assertEqual(cells, exported_cells)
def do_import_lines(self, lo, hi, key, time=1408604686):
lines = [
str.format("GSM,{mcc},{mnc},{lac},{cid}," +
",{lon},{lat},1,1,1,{time},{time},",
cid=i * 1010,
lon=PARIS_LON + i * 0.002,
lat=PARIS_LAT + i * 0.001,
time=time, **key)
for i in range(lo, hi)]
txt = "\n".join(lines)
with selfdestruct_tempdir() as d:
path = os.path.join(d, "import.csv.gz")
with GzipFile(path, 'wb') as f:
f.write(txt)
import_ocid_cells(path, sess=self.db_slave_session)
def get_test_csv(self, lo=1, hi=10, time=1408604686):
line_template = ('GSM,{mcc},{mnc},{lac},{cid},,{lon},'
'{lat},1,1,1,{time},{time},')
lines = [line_template.format(
cid=i * 1010,
lon=PARIS_LON + i * 0.002,
lat=PARIS_LAT + i * 0.001,
time=time,
**self.KEY)
for i in range(lo, hi)]
txt = '\n'.join(lines)
with selfdestruct_tempdir() as d:
path = os.path.join(d, 'import.csv.gz')
with GzipFile(path, 'wb') as f:
f.write(txt)
yield path
def get_test_csv(self, lo=1, hi=10, time=1408604686):
line_template = ('GSM,{mcc},{mnc},{lac},{cid},,{lon},'
'{lat},1,1,1,{time},{time},')
lines = [
line_template.format(cid=i * 1010,
lon=PARIS_LON + i * 0.002,
lat=PARIS_LAT + i * 0.001,
time=time,
**self.KEY) for i in range(lo, hi)
]
txt = '\n'.join(lines)
with selfdestruct_tempdir() as d:
path = os.path.join(d, 'import.csv.gz')
with GzipFile(path, 'wb') as f:
f.write(txt)
yield path
def test_local_export(self):
session = self.db_master_session
cell_fixture_fields = ('radio', 'cid', 'lat', 'lon', 'mnc', 'mcc',
'lac')
cell_key = {'radio': RADIO_TYPE['gsm'], 'mcc': 1, 'mnc': 2, 'lac': 4}
cells = set()
for cid in range(190, 200):
cell = dict(cid=cid, lat=1.0, lon=2.0, **cell_key)
session.add(Cell(**cell))
cell['radio'] = 'GSM'
cell_strings = [(field, str(value))
for (field, value) in cell.items()]
cell_tuple = tuple(sorted(cell_strings))
cells.add(cell_tuple)
# add one incomplete / unprocessed cell
session.add(Cell(cid=210, lat=None, lon=None, **cell_key))
session.commit()
with selfdestruct_tempdir() as temp_dir:
path = os.path.join(temp_dir, 'export.csv.gz')
cond = cell_table.c.lat.isnot(None)
write_stations_to_csv(session, cell_table, CELL_COLUMNS, cond,
path, make_cell_export_dict, CELL_FIELDS)
with GzipFile(path, 'rb') as gzip_file:
reader = csv.DictReader(gzip_file, CELL_FIELDS)
header = reader.next()
self.assertTrue('area' in header.values())
self.assertEqual(header, CELL_HEADER_DICT)
exported_cells = set()
for exported_cell in reader:
exported_cell_filtered = [
(field, value)
for (field, value) in exported_cell.items()
if field in cell_fixture_fields
]
exported_cell = tuple(sorted(exported_cell_filtered))
exported_cells.add(exported_cell)
self.assertEqual(cells, exported_cells)
def do_import_lines(self, lo, hi, key, time=1408604686):
lines = [
str.format("GSM,{mcc},{mnc},{lac},{cid}," +
",{lon},{lat},1,1,1,{time},{time},",
cid=i * 1010,
lon=PARIS_LON + i * 0.002,
lat=PARIS_LAT + i * 0.001,
time=time,
**key) for i in range(lo, hi)
]
txt = "\n".join(lines)
with selfdestruct_tempdir() as d:
path = os.path.join(d, "import.csv.gz")
with GzipFile(path, 'wb') as f:
f.write(txt)
import_ocid_cells(path, sess=self.db_slave_session)
def local_test_csv_file(self):
txt = """\
GSM,302,2,4,190,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,191,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,192,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,193,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,194,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,195,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,196,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,197,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,198,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,199,,2.0,1.0,0,0,1,1408604686,1408604686,
"""
with selfdestruct_tempdir() as d:
path = os.path.join(d, "import.csv.gz")
with GzipFile(path, 'wb') as f:
f.write(txt)
f.flush()
yield path
def local_test_csv_file(self):
txt = """\
radio,mcc,net,area,cell,unit,lon,lat,range,samples,changeable,created,updated,
GSM,302,2,4,190,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,191,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,192,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,193,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,194,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,195,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,196,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,197,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,198,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,199,,2.0,1.0,0,0,1,1408604686,1408604686,
"""
with selfdestruct_tempdir() as d:
path = os.path.join(d, "import.csv.gz")
with GzipFile(path, 'wb') as f:
f.write(txt)
f.flush()
yield path
def local_test_csv_file(self):
txt = """\
radio,mcc,net,area,cell,unit,lon,lat,range,samples,changeable,created,updated,
GSM,302,2,4,190,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,191,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,192,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,193,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,194,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,195,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,196,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,197,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,198,,2.0,1.0,0,0,1,1408604686,1408604686,
GSM,302,2,4,199,,2.0,1.0,0,0,1,1408604686,1408604686,
"""
with selfdestruct_tempdir() as d:
path = os.path.join(d, "import.csv.gz")
with GzipFile(path, 'wb') as f:
f.write(txt)
f.flush()
yield path