def test_resolve_xref_gets_id_for_xref_field_ending_with_id(self, cache):
cache.clear_cache()
data = {'country_id': 'NG'}
session = cache._XRefResolver__dbsession
imp = ValueReader(AttrDict(wb=None, db=session, cache=cache))
imp.resolve_xref(data, ('country_id', 'code', Country))
assert type(data['country_id']) is uuid.UUID
def test_resolve_xref_gets_model_for_xref_field_ending_witout_id(self, cache):
cache.clear_cache()
data = {'country': 'NG'}
session = cache._XRefResolver__dbsession
imp = ValueReader(AttrDict(wb=None, db=session, cache=cache))
imp.resolve_xref(data, ('country', 'code', Country))
assert isinstance(data['country'], Country)
def __init__(self, context, matcher=None):
assert 'db' in context
if not isinstance(context, AttrDict):
context = AttrDict(context)
self.matcher = matcher or PrefixedTextMatcher()
self.context = context
self.errors = []
def import_line(self, sh, row):
num_errors = len(self.errors)
subtype = self.get_required_enum_from_cell(sh, row, 2, LineType)
if num_errors < len(self.errors):
return (row + 1)
voltage = self.get_required_id_from_cell(sh, row+1, 2)
if num_errors < len(self.errors):
return
row += 3
while row <= sh.max_row:
if self.is_empty_row(sh, row):
break
num_errors = len(self.errors)
data = AttrDict({'subtype': subtype, 'voltage_id': voltage})
data['source_station_id'] = self.get_required_id_from_cell(sh, row, 1)
data['line_code'] = self.get_required_id_from_cell(sh, row, 2)
data['register_code'] = self.get_required_id_from_cell(sh, row, 3)
data['organisation'] = self.get_ids_from_cell(sh, row, 4)
data['name'] = self.get_required_text_from_cell(sh, row, 5)
data['is_public'] = self.get_required_bool_from_cell(sh, row, 6)
data['owner'] = self.get_required_enum_from_cell(sh, row, 7, Owner)
data['date_commissioned'] = self.get_date_from_cell(sh, row, 8)
data['comments'] = self.get_text_from_cell(sh, row, 9, None)
if num_errors < len(self.errors):
break
self.create_line(row, data)
row += 1
return (row + 1)
def imp_sl():
def initdb(session):
cn = Country(code='CN', name='Country')
org = Organisation(identifier="Org0",
name="Org.Main",
short_name="Org0")
session.add_all([
## voltages
Voltage(value=415),
Voltage(value=11000),
Voltage(value=33000),
## states
State(code='S1', name='State 1', country=cn),
State(code='S2', name='State 2', country=cn),
## organisations
Organisation(identifier="Org1",
name="Child1",
short_name="Org1",
parent=org),
Organisation(identifier="Org2",
name="Child2",
short_name="Org2",
parent=org),
])
session.commit()
## setup
resx = utils.make_session(initdb_callback=initdb)
cache = XRefResolver(resx.session)
return StationLineImporter(AttrDict(db=resx.session, cache=cache))
## teardown
utils.drop_tables(resx.engine)
def __init__(self, context, progress_callback=None):
assert 'db' in context
assert 'cache' in context
self.progress_callback = progress_callback
if not isinstance(context, AttrDict):
context = AttrDict(context)
self.context = context
self.errors = []
def process(self):
sh, nrows = (self.sheet, self.sheet.max_row)
for row in range(2, nrows + 1):
data = AttrDict()
num_errors = len(self.errors)
data['value'] = self.get_required_int_from_cell(sh, row, 1)
if num_errors == len(self.errors):
self.create_voltage(row, data)
self.progress(row, nrows)
def as_attrd(self):
return AttrDict({
'addr_raw': self.raw,
'addr_street': self.street,
'addr_town': self.town,
'addr_state': self.state,
'addr_landmark': self.landmark,
'postal_code': self.postal_code,
})
def process(self):
sh, nrows = (self.sheet, self.sheet.max_row)
for row in range(2, nrows + 1):
data = AttrDict()
num_errors = len(self.errors)
self.process_chunk(row, 1, data)
if num_errors == len(self.errors):
item = self.create_item(row, data)
self.add_item(row, item)
self.progress(row, nrows)
self.post_process()
self.progress(row, nrows)
def test_organization_type_import(self, cache):
from conftest import wb
db = cache._XRefResolver__dbsession
utils.clear_tables(db, 'organization_types')
context = AttrDict(db=db, cache=cache)
importer = OrganizationTypeImporter(context)
importer.import_data(wb())
assert len(importer.errors) == 0
found = db.query(OrganizationType).count()
assert found == 3
def process(self):
sh, nrows, num_errors = (self.sheet, self.sheet.max_row, 0)
for row in range(2, nrows + 1):
data = AttrDict()
data['name'] = self.get_required_id_from_cell(sh, row, 1)
data['title'] = self.get_required_text_from_cell(sh, row, 2)
data['is_root'] = self.get_bool_from_cell(sh, row, 3)
if num_errors <= len(self.errors):
self.create_organization_type(row, data)
self.progress(row, nrows)
num_errors = len(self.errors)
row += 1
self.progress(row, nrows)
def _process(db, options):
# check that the file exists
if not os.path.exists(options['file']):
print('File not found: %s' % options['file'])
sys.exit(0)
from openpyxl import load_workbook
wb = load_workbook(options['file'], read_only=True)
importer = MegaImporter(AttrDict(db=db, cache=XRefResolver(db)))
importer.import_data(wb)
if importer.has_errors:
_log_errors(importer.summarise_errors(),
*os.path.split(options['file']))
print('View error log for details...')
def import_country(self, sh, row):
row, nrows = (row + 2, sh.max_row)
num_errors = len(self.errors)
while row <= nrows:
if self.is_empty_row(sh, row):
break
data = AttrDict()
data['code'] = self.get_required_id_from_cell(sh, row, 1)
data['name'] = self.get_required_text_from_cell(sh, row, 2)
if num_errors < len(self.errors):
break
self.create_country(row, data)
self.progress(row, nrows)
row += 1
return (row + 1)
def test_organization_type_fails_with_multi_isroot_non_multiroot(self, cache):
from conftest import wb
OrganizationTypeImporter.sheet_name = 'organization-types-2'
db = cache._XRefResolver__dbsession
utils.clear_tables(db, 'organization_types')
context = AttrDict(db=db, cache=cache)
importer = OrganizationTypeImporter(context)
importer.import_data(wb())
assert len(importer.errors) == 1
found = db.query(OrganizationType).count()
assert found == 2
# restore orig sheet name
OrganizationTypeImporter.sheet_name = 'organization-types'
def test_organisations_import(self, cache2):
from enum import Enum
from conftest import wb
db = cache2._XRefResolver__dbsession
utils.clear_tables(db, 'contact_details', 'organizations')
context = AttrDict(db=db, cache=cache2)
importer = OrganizationImporter(context)
importer.import_data(wb())
assert len(importer.errors) == 0
found = db.query(Organization).count()
assert found == 2
found2 = db.query(EmailContact).count()
assert found2 == 2
found3 = db.query(PhoneContact).count()
assert found3 == 1
def import_station(self, sh, row):
num_errors = len(self.errors)
subtype = self.get_required_enum_from_cell(sh, row, 2, StationType)
if num_errors < len(self.errors):
return (row + 1)
row += 2
while row <= sh.max_row:
if self.is_empty_row(sh, row):
break
data, num_errors = (AttrDict({'subtype': subtype}), len(self.errors))
if subtype != StationType.transmission:
data['source_line_id'] = self.get_required_id_from_cell(sh, row, 1)
data['facility_code'] = self.get_required_id_from_cell(sh, row, 2)
data['register_code'] = self.get_required_id_from_cell(sh, row, 3)
data['organisation_id'] = self.get_id_from_cell(sh, row, 4)
data['name'] = self.get_required_text_from_cell(sh, row, 5)
data['is_public'] = self.get_required_bool_from_cell(sh, row, 6)
data['is_manned'] = self.get_required_bool_from_cell(sh, row, 7)
data['owner'] = self.get_required_enum_from_cell(sh, row, 8, Owner)
data['addr_state_id'] = self.get_required_text_from_cell(sh, row, 9)
data['addr_street'] = self.get_text_from_cell(sh, row, 10, None)
data['addr_town'] = self.get_text_from_cell(sh, row, 11, None)
data['addr_landmark'] = self.get_text_from_cell(sh, row, 12, None)
data['longitude'] = self.get_float_from_cell(sh, row, 13, None)
data['latitude'] = self.get_float_from_cell(sh, row, 14, None)
data['altitude'] = self.get_float_from_cell(sh, row, 15, None)
data['gps_error'] = self.get_int_from_cell(sh, row, 16, None)
data['date_installed'] = self.get_date_from_cell(sh, row, 17)
data['comments'] = self.get_text_from_cell(sh, row, 18, None)
if num_errors < len(self.errors):
break
self.create_station(row, data)
row += 1
return (row + 1)
def __init__(self, context):
self.context = AttrDict(context)
self.wb = context.pop('wb')