def _moveAttribs(self):
"""
Some of item attributes are defined in invtypes table.
We do not need them there, for data consistency it's worth
to move them to dgmtypeattribs table.
"""
atrribMap = {
'radius': Attribute.radius,
'mass': Attribute.mass,
'volume': Attribute.volume,
'capacity': Attribute.capacity
}
attrIds = tuple(atrribMap.values())
# Here we will store pairs (typeID, attrID) already
# defined in table
definedPairs = set()
dgmtypeattribs = self.data['dgmtypeattribs']
for row in dgmtypeattribs:
if row['attributeID'] not in attrIds:
continue
definedPairs.add((row['typeID'], row['attributeID']))
attrsSkipped = 0
newInvtypes = set()
# Cycle through all invtypes, for each row moving each its field
# either to different table or container for updated rows
for row in self.data['invtypes']:
typeId = row['typeID']
newRow = {}
for field, value in row.items():
if field in atrribMap:
# If row didn't have such attribute defined, skip it
if value is None:
continue
# If such attribute already exists in dgmtypeattribs,
# do not modify it - values from dgmtypeattribs table
# have priority
attrId = atrribMap[field]
if (typeId, attrId) in definedPairs:
attrsSkipped += 1
continue
# Generate row and add it to proper attribute table
dgmtypeattribs.add(
frozendict({
'typeID': typeId,
'attributeID': attrId,
'value': value
}))
else:
newRow[field] = value
newInvtypes.add(frozendict(newRow))
# Update invtypes with rows which do not contain attributes
self.data['invtypes'].clear()
self.data['invtypes'].update(newInvtypes)
if attrsSkipped > 0:
msg = '{} built-in attributes already have had value in dgmtypeattribs and were skipped'.format(
attrsSkipped)
self._logger.warning(msg, childName='cacheGenerator')
def _moveAttribs(self):
"""
Some of item attributes are defined in invtypes table.
We do not need them there, for data consistency it's worth
to move them to dgmtypeattribs table.
"""
atrribMap = {
'radius': Attribute.radius,
'mass': Attribute.mass,
'volume': Attribute.volume,
'capacity': Attribute.capacity
}
attrIds = tuple(atrribMap.values())
# Here we will store pairs (typeID, attrID) already
# defined in table
definedPairs = set()
dgmtypeattribs = self.data['dgmtypeattribs']
for row in dgmtypeattribs:
if row['attributeID'] not in attrIds:
continue
definedPairs.add((row['typeID'], row['attributeID']))
attrsSkipped = 0
newInvtypes = set()
# Cycle through all invtypes, for each row moving each its field
# either to different table or container for updated rows
for row in self.data['invtypes']:
typeId = row['typeID']
newRow = {}
for field, value in row.items():
if field in atrribMap:
# If row didn't have such attribute defined, skip it
if value is None:
continue
# If such attribute already exists in dgmtypeattribs,
# do not modify it - values from dgmtypeattribs table
# have priority
attrId = atrribMap[field]
if (typeId, attrId) in definedPairs:
attrsSkipped += 1
continue
# Generate row and add it to proper attribute table
dgmtypeattribs.add(
frozendict({
'typeID': typeId,
'attributeID': attrId,
'value': value
}))
else:
newRow[field] = value
newInvtypes.add(frozendict(newRow))
# Update invtypes with rows which do not contain attributes
self.data['invtypes'].clear()
self.data['invtypes'].update(newInvtypes)
if attrsSkipped > 0:
msg = '{} built-in attributes already have had value in dgmtypeattribs and were skipped'.format(
attrsSkipped)
self._logger.warning(msg, childName='cacheGenerator')
def _multiple_default_effects(dte_rows):
"""Check that each type has one default effect maximum."""
# Set with IDs of item types, which have default effect
defeff_type_ids = set()
invalid_rows = set()
for row in sorted(dte_rows, key=lambda r: r['table_pos']):
is_default = row.get('isDefault')
# We're interested only in default effects
if not is_default:
continue
type_id = row['typeID']
# If we already saw default effect for given type ID, invalidate
# current row
if type_id in defeff_type_ids:
invalid_rows.add(row)
else:
defeff_type_ids.add(type_id)
if invalid_rows:
msg = ('data contains {} excessive default effects, '
'marking them as non-default').format(len(invalid_rows))
logger.warning(msg)
# Replace isDefault field value with False for invalid rows
dte_rows.difference_update(invalid_rows)
for invalid_row in invalid_rows:
new_row = {}
for field, value in invalid_row.items():
new_row[field] = False if field == 'isDefault' else value
dte_rows.add(frozendict(new_row))
def _multipleDefaultEffects(self):
"""
Each type must have one default effect at max. Data
conversion (and resulting data structure) relies on
this assumption.
"""
# Set with IDs of types, which have default effect
defeff = set()
table = self.data['dgmtypeeffects']
invalidRows = set()
for row in sorted(table, key=lambda row: row['tablePos']):
isDefault = row.get('isDefault')
# We're interested only in default effects
if isDefault is not True:
continue
typeId = row['typeID']
# If we already saw default effect for given type ID,
# invalidate current row
if typeId in defeff:
invalidRows.add(row)
else:
defeff.add(typeId)
# Process ivalid rows, if any
if invalidRows:
msg = 'data contains {} excessive default effects, marking them as non-default'.format(len(invalidRows))
self._logger.warning(msg, childName='cacheGenerator')
# Replace isDefault field value with False for invalid rows
table.difference_update(invalidRows)
for invalidRow in invalidRows:
newRow = {}
for field, value in invalidRow.items():
newRow[field] = False if field == 'isDefault' else value
table.add(frozendict(newRow))
def _multipleDefaultEffects(self):
"""
Each type must have one default effect at max. Data
conversion (and resulting data structure) relies on
this assumption.
"""
# Set with IDs of types, which have default effect
defeff = set()
table = self.data['dgmtypeeffects']
invalidRows = set()
for row in sorted(table, key=lambda row: row['tablePos']):
isDefault = row.get('isDefault')
# We're interested only in default effects
if isDefault is not True:
continue
typeId = row['typeID']
# If we already saw default effect for given type ID,
# invalidate current row
if typeId in defeff:
invalidRows.add(row)
else:
defeff.add(typeId)
# Process ivalid rows, if any
if invalidRows:
msg = 'data contains {} excessive default effects, marking them as non-default'.format(
len(invalidRows))
self._logger.warning(msg, childName='cacheGenerator')
# Replace isDefault field value with False for invalid rows
table.difference_update(invalidRows)
for invalidRow in invalidRows:
newRow = {}
for field, value in invalidRow.items():
newRow[field] = False if field == 'isDefault' else value
table.add(frozendict(newRow))
def _multiple_default_effects(dte_rows):
"""Check that each type has one default effect maximum."""
# Set with IDs of item types, which have default effect
defeff_type_ids = set()
invalid_rows = set()
for row in sorted(dte_rows, key=lambda r: r['table_pos']):
is_default = row.get('isDefault')
# We're interested only in default effects
if not is_default:
continue
type_id = row['typeID']
# If we already saw default effect for given type ID, invalidate
# current row
if type_id in defeff_type_ids:
invalid_rows.add(row)
else:
defeff_type_ids.add(type_id)
if invalid_rows:
msg = (
'data contains {} excessive default effects, '
'marking them as non-default'
).format(len(invalid_rows))
logger.warning(msg)
# Replace isDefault field value with False for invalid rows
dte_rows.difference_update(invalid_rows)
for invalid_row in invalid_rows:
new_row = {}
for field, value in invalid_row.items():
new_row[field] = False if field == 'isDefault' else value
dte_rows.add(frozendict(new_row))
def _freeze_data(cls, data):
if isinstance(data, dict):
return frozendict({
cls._freeze_data(k): cls._freeze_data(v)
for k, v
in data.items()})
if isinstance(data, list):
return tuple([cls._freeze_data(d) for d in data])
if isinstance(data, set):
return frozenset([cls._freeze_data(d) for d in data])
return data
def _freezeModifier(self, modifier):
"""
Converts modifier into frozendict with its keys and
values assigned according to modifier's ones.
"""
# Fields which we need to dump into row
fields = ('state', 'context', 'sourceAttributeId', 'operator',
'targetAttributeId', 'location', 'filterType', 'filterValue')
modifierRow = {}
for field in fields:
modifierRow[field] = getattr(modifier, field)
frozenRow = frozendict(modifierRow)
return frozenRow
def _freezeModifier(self, modifier):
"""
Converts modifier into frozendict with its keys and
values assigned according to modifier's ones.
"""
# Fields which we need to dump into row
fields = ('state', 'context', 'sourceAttributeId', 'operator',
'targetAttributeId', 'location', 'filterType', 'filterValue')
modifierRow = {}
for field in fields:
modifierRow[field] = getattr(modifier, field)
frozenRow = frozendict(modifierRow)
return frozenRow
def _move_attrs(data):
"""Normalize attribute value definitions.
Some of item type attributes are defined in evetypes table. We do not
need them there, for data consistency it's worth to move them to
dgmtypeattribs table, where the rest of attributes are defined.
Args:
data: Dictionary in {table name: {table, rows}} format.
"""
attr_map = {
'radius': AttrId.radius,
'mass': AttrId.mass,
'volume': AttrId.volume,
'capacity': AttrId.capacity
}
attr_ids = tuple(attr_map.values())
# Here we will store pairs (typeID, attrID) already defined in
# dgmtypeattribs
defined_pairs = set()
dgmtypeattribs = data['dgmtypeattribs']
for row in dgmtypeattribs:
if row['attributeID'] not in attr_ids:
continue
defined_pairs.add((row['typeID'], row['attributeID']))
attrs_skipped = 0
for row in data['evetypes']:
type_id = row['typeID']
for field, value in row.items():
if field in attr_map:
# If row didn't have such attribute defined, skip it
if value is None:
continue
# If such attribute already exists in dgmtypeattribs, do not
# modify it - values from dgmtypeattribs table have priority
attr_id = attr_map[field]
if (type_id, attr_id) in defined_pairs:
attrs_skipped += 1
continue
# Generate row and add it to proper attribute table
dgmtypeattribs.add(
frozendict({
'typeID': type_id,
'attributeID': attr_id,
'value': value
}))
if attrs_skipped:
msg = ('{} built-in attributes already have had value '
'in dgmtypeattribs and were skipped').format(attrs_skipped)
logger.warning(msg)
def _move_attrs(data):
"""Normalize attribute value definitions.
Some of item type attributes are defined in evetypes table. We do not
need them there, for data consistency it's worth to move them to
dgmtypeattribs table, where the rest of attributes are defined.
Args:
data: Dictionary in {table name: {table, rows}} format.
"""
attr_map = {
'radius': AttrId.radius,
'mass': AttrId.mass,
'volume': AttrId.volume,
'capacity': AttrId.capacity}
attr_ids = tuple(attr_map.values())
# Here we will store pairs (typeID, attrID) already defined in
# dgmtypeattribs
defined_pairs = set()
dgmtypeattribs = data['dgmtypeattribs']
for row in dgmtypeattribs:
if row['attributeID'] not in attr_ids:
continue
defined_pairs.add((row['typeID'], row['attributeID']))
attrs_skipped = 0
for row in data['evetypes']:
type_id = row['typeID']
for field, value in row.items():
if field in attr_map:
# If row didn't have such attribute defined, skip it
if value is None:
continue
# If such attribute already exists in dgmtypeattribs, do not
# modify it - values from dgmtypeattribs table have priority
attr_id = attr_map[field]
if (type_id, attr_id) in defined_pairs:
attrs_skipped += 1
continue
# Generate row and add it to proper attribute table
dgmtypeattribs.add(frozendict({
'typeID': type_id,
'attributeID': attr_id,
'value': value}))
if attrs_skipped:
msg = (
'{} built-in attributes already have had value '
'in dgmtypeattribs and were skipped'
).format(attrs_skipped)
logger.warning(msg)
def run(self, dataHandler):
"""
Generate cache out of passed data.
Positional arguments:
dataHandler - data handler to use for getting data
Return value:
Dictionary in {entity type: [{field name: field value}]
format
"""
# Put all the data we need into single dictionary
# Format, as usual, {table name: table}, where table
# is set of rows, which are represented by frozendicts
# {fieldName: fieldValue}. Combination of sets and
# frozendicts is used to speed up several stages of
# the generator.
data = {}
tables = {'invtypes': dataHandler.getInvtypes,
'invgroups': dataHandler.getInvgroups,
'dgmattribs': dataHandler.getDgmattribs,
'dgmtypeattribs': dataHandler.getDgmtypeattribs,
'dgmeffects': dataHandler.getDgmeffects,
'dgmtypeeffects': dataHandler.getDgmtypeeffects,
'dgmexpressions': dataHandler.getDgmexpressions}
for tablename, method in tables.items():
tablePos = 0
# For faster processing of various operations,
# freeze table rows and put them into set
table = set()
for row in method():
# During further generator stages. some of rows
# may fall in risk groups, where all rows but one
# need to be removed. To deterministically remove rows
# based on position in original data, write position
# to each row
row['tablePos'] = tablePos
tablePos += 1
table.add(frozendict(row))
data[tablename] = table
# Run pre-cleanup checks, as cleaning and further stages
# rely on some assumptions about the data
self._checker.preCleanup(data)
# Also normalize the data to make data structure
# more consistent, and thus easier to clean properly
self._converter.normalize(data)
# Clean our container out of unwanted data
self._cleaner.clean(data)
# Verify that our data is ready for conversion
self._checker.preConvert(data)
# Convert data into Eos-specific format. Here tables are
# no longer represented by sets of frozendicts, but by
# list of dicts
data = self._converter.convert(data)
return data
def _convert_expression_symbolic_references(data):
"""Convert all known symbolic references to int references.
Some of entities in dgmexpressions table are defined not as IDs, but as
symbolic references. CCP most likely has these symbolic names defined in
their code, thus we have to hardcode it too.
Args:
data: Dictionary in {table name: {table, rows}} format.
"""
dgmexps = data['dgmexpressions']
# Replacement specification
# Format:
# (
# (
# operator,
# column name for entity ID,
# {replacement: map},
# (ignored names, ...)
# ),
# ...
# )
repl_spec = ((OperandId.def_attr, 'expressionAttributeID', {},
('shieldDamage', )),
(OperandId.def_grp, 'expressionGroupID', {
'EnergyWeapon': TypeGroupId.energy_weapon,
'HybridWeapon': TypeGroupId.hydrid_weapon,
'MiningLaser': TypeGroupId.mining_laser,
'ProjectileWeapon': TypeGroupId.projectile_weapon
}, ('Structure', 'PowerCore',
' None')), (OperandId.def_type, 'expressionTypeID',
{}, ('Acceration Control', )))
for operand, id_col_name, repls, ignored_names in repl_spec:
used_repls = set()
unknown_names = set()
# We're modifying only rows with specific operands
for exp_row in dgmexps:
if exp_row['operandID'] != operand:
continue
exp_entity_id = exp_row[id_col_name]
# If entity is already referenced via ID, nothing to do here
if exp_entity_id is not None:
continue
symbolic_entity_name = exp_row['expressionValue']
# Skip names we've set to ignore explicitly
if symbolic_entity_name in ignored_names:
continue
# Do replacements if they're known to us
if symbolic_entity_name in repls:
# As rows are frozen dicts, we compose new mutable dict,
# update data there, freeze it, and do actual replacement
new_exp_row = {}
new_exp_row.update(exp_row)
new_exp_row['expressionValue'] = None
new_exp_row[id_col_name] = repls[symbolic_entity_name]
new_exp_row = frozendict(new_exp_row)
dgmexps.remove(exp_row)
dgmexps.add(new_exp_row)
used_repls.add(symbolic_entity_name)
continue
# If we do not know it, add to special container which we will
# use later
unknown_names.add(symbolic_entity_name)
# Report results to log, it will help to indicate when CCP finally
# stops using literal references, and we can get rid of this
# conversion, or add some new
unused_repls = set(repls).difference(used_repls)
if unused_repls:
unused_repls = sorted(unused_repls)
unused_line = ', '.join('"{}"'.format(r) for r in unused_repls)
msg = '{} replacements for {} were not used: {}'.format(
len(unused_repls), id_col_name, unused_line)
logger.warning(msg)
if unknown_names:
unknown_names = sorted(unknown_names)
unknown_line = ', '.join('"{}"'.format(n)
for n in unknown_names)
msg = ('unable to convert {} literal references to {}: {}'
).format(len(unknown_names), id_col_name, unknown_line)
logger.warning(msg)
def run(data_handler):
"""Run eve object building process.
Use data provided by passed cache handler to compose various objects
with the help of which eos will oeprate.
Args:
data_handler: Data handler instance, which should provide access to
raw eve data.
Returns:
3 iterables, which contain types, attributes and effects.
"""
# Put all the data we need into single dictionary Format, as usual,
# {table name: table}, where table is set of rows, which are
# represented by frozendicts {fieldName: fieldValue}. Combination of
# sets and frozendicts is used to speed up several stages of the
# builder.
data = {}
getter_map = {
'evetypes': data_handler.get_evetypes,
'evegroups': data_handler.get_evegroups,
'dgmattribs': data_handler.get_dgmattribs,
'dgmtypeattribs': data_handler.get_dgmtypeattribs,
'dgmeffects': data_handler.get_dgmeffects,
'dgmtypeeffects': data_handler.get_dgmtypeeffects,
'dgmexpressions': data_handler.get_dgmexpressions,
'typefighterabils': data_handler.get_typefighterabils
}
for table_name, getter in getter_map.items():
table_pos = 0
table = set()
for row in getter():
# During further builder stages. some of rows may fall in risk
# groups, where all rows but one need to be removed. To
# deterministically remove rows based on position in original
# data, write position to each row
row['table_pos'] = table_pos
table_pos += 1
table.add(frozendict(row))
data[table_name] = table
# Run pre-cleanup checks, as cleanup stage and further stages rely on
# some assumptions about the data
ValidatorPreClean.run(data)
# Normalize the data to make data structure more consistent, making it
# easier to clean properly
Normalizer.run(data)
# Remove unwanted data
Cleaner().clean(data)
# Verify that our data is ready for conversion
ValidatorPreConv.run(data)
# Convert data into Eos-specific objects
types, attrs, effects = Converter.run(data)
return types, attrs, effects
def _convert_expression_symbolic_references(data):
"""Convert all known symbolic references to int references.
Some of entities in dgmexpressions table are defined not as IDs, but as
symbolic references. CCP most likely has these symbolic names defined in
their code, thus we have to hardcode it too.
Args:
data: Dictionary in {table name: {table, rows}} format.
"""
dgmexps = data['dgmexpressions']
# Replacement specification
# Format:
# (
# (
# operator,
# column name for entity ID,
# {replacement: map},
# (ignored names, ...)
# ),
# ...
# )
repl_spec = (
(
OperandId.def_attr,
'expressionAttributeID',
{},
('shieldDamage',)),
(
OperandId.def_grp,
'expressionGroupID',
{
'EnergyWeapon': TypeGroupId.energy_weapon,
'HybridWeapon': TypeGroupId.hydrid_weapon,
'MiningLaser': TypeGroupId.mining_laser,
'ProjectileWeapon': TypeGroupId.projectile_weapon},
('Structure', 'PowerCore', ' None')),
(
OperandId.def_type,
'expressionTypeID',
{},
('Acceration Control',)))
for operand, id_col_name, repls, ignored_names in repl_spec:
used_repls = set()
unknown_names = set()
# We're modifying only rows with specific operands
for exp_row in dgmexps:
if exp_row['operandID'] != operand:
continue
exp_entity_id = exp_row[id_col_name]
# If entity is already referenced via ID, nothing to do here
if exp_entity_id is not None:
continue
symbolic_entity_name = exp_row['expressionValue']
# Skip names we've set to ignore explicitly
if symbolic_entity_name in ignored_names:
continue
# Do replacements if they're known to us
if symbolic_entity_name in repls:
# As rows are frozen dicts, we compose new mutable dict,
# update data there, freeze it, and do actual replacement
new_exp_row = {}
new_exp_row.update(exp_row)
new_exp_row['expressionValue'] = None
new_exp_row[id_col_name] = repls[symbolic_entity_name]
new_exp_row = frozendict(new_exp_row)
dgmexps.remove(exp_row)
dgmexps.add(new_exp_row)
used_repls.add(symbolic_entity_name)
continue
# If we do not know it, add to special container which we will
# use later
unknown_names.add(symbolic_entity_name)
# Report results to log, it will help to indicate when CCP finally
# stops using literal references, and we can get rid of this
# conversion, or add some new
unused_repls = set(repls).difference(used_repls)
if unused_repls:
unused_repls = sorted(unused_repls)
unused_line = ', '.join('"{}"'.format(r) for r in unused_repls)
msg = '{} replacements for {} were not used: {}'.format(
len(unused_repls), id_col_name, unused_line)
logger.warning(msg)
if unknown_names:
unknown_names = sorted(unknown_names)
unknown_line = ', '.join(
'"{}"'.format(n) for n in unknown_names)
msg = (
'unable to convert {} literal references to {}: {}'
).format(len(unknown_names), id_col_name, unknown_line)
logger.warning(msg)
