def _moveAttribs(self): """ Some of item attributes are defined in invtypes table. We do not need them there, for data consistency it's worth to move them to dgmtypeattribs table. """ atrribMap = { 'radius': Attribute.radius, 'mass': Attribute.mass, 'volume': Attribute.volume, 'capacity': Attribute.capacity } attrIds = tuple(atrribMap.values()) # Here we will store pairs (typeID, attrID) already # defined in table definedPairs = set() dgmtypeattribs = self.data['dgmtypeattribs'] for row in dgmtypeattribs: if row['attributeID'] not in attrIds: continue definedPairs.add((row['typeID'], row['attributeID'])) attrsSkipped = 0 newInvtypes = set() # Cycle through all invtypes, for each row moving each its field # either to different table or container for updated rows for row in self.data['invtypes']: typeId = row['typeID'] newRow = {} for field, value in row.items(): if field in atrribMap: # If row didn't have such attribute defined, skip it if value is None: continue # If such attribute already exists in dgmtypeattribs, # do not modify it - values from dgmtypeattribs table # have priority attrId = atrribMap[field] if (typeId, attrId) in definedPairs: attrsSkipped += 1 continue # Generate row and add it to proper attribute table dgmtypeattribs.add( frozendict({ 'typeID': typeId, 'attributeID': attrId, 'value': value })) else: newRow[field] = value newInvtypes.add(frozendict(newRow)) # Update invtypes with rows which do not contain attributes self.data['invtypes'].clear() self.data['invtypes'].update(newInvtypes) if attrsSkipped > 0: msg = '{} built-in attributes already have had value in dgmtypeattribs and were skipped'.format( attrsSkipped) self._logger.warning(msg, childName='cacheGenerator')
def _multiple_default_effects(dte_rows): """Check that each type has one default effect maximum.""" # Set with IDs of item types, which have default effect defeff_type_ids = set() invalid_rows = set() for row in sorted(dte_rows, key=lambda r: r['table_pos']): is_default = row.get('isDefault') # We're interested only in default effects if not is_default: continue type_id = row['typeID'] # If we already saw default effect for given type ID, invalidate # current row if type_id in defeff_type_ids: invalid_rows.add(row) else: defeff_type_ids.add(type_id) if invalid_rows: msg = ('data contains {} excessive default effects, ' 'marking them as non-default').format(len(invalid_rows)) logger.warning(msg) # Replace isDefault field value with False for invalid rows dte_rows.difference_update(invalid_rows) for invalid_row in invalid_rows: new_row = {} for field, value in invalid_row.items(): new_row[field] = False if field == 'isDefault' else value dte_rows.add(frozendict(new_row))
def _multipleDefaultEffects(self): """ Each type must have one default effect at max. Data conversion (and resulting data structure) relies on this assumption. """ # Set with IDs of types, which have default effect defeff = set() table = self.data['dgmtypeeffects'] invalidRows = set() for row in sorted(table, key=lambda row: row['tablePos']): isDefault = row.get('isDefault') # We're interested only in default effects if isDefault is not True: continue typeId = row['typeID'] # If we already saw default effect for given type ID, # invalidate current row if typeId in defeff: invalidRows.add(row) else: defeff.add(typeId) # Process ivalid rows, if any if invalidRows: msg = 'data contains {} excessive default effects, marking them as non-default'.format(len(invalidRows)) self._logger.warning(msg, childName='cacheGenerator') # Replace isDefault field value with False for invalid rows table.difference_update(invalidRows) for invalidRow in invalidRows: newRow = {} for field, value in invalidRow.items(): newRow[field] = False if field == 'isDefault' else value table.add(frozendict(newRow))
def _multipleDefaultEffects(self): """ Each type must have one default effect at max. Data conversion (and resulting data structure) relies on this assumption. """ # Set with IDs of types, which have default effect defeff = set() table = self.data['dgmtypeeffects'] invalidRows = set() for row in sorted(table, key=lambda row: row['tablePos']): isDefault = row.get('isDefault') # We're interested only in default effects if isDefault is not True: continue typeId = row['typeID'] # If we already saw default effect for given type ID, # invalidate current row if typeId in defeff: invalidRows.add(row) else: defeff.add(typeId) # Process ivalid rows, if any if invalidRows: msg = 'data contains {} excessive default effects, marking them as non-default'.format( len(invalidRows)) self._logger.warning(msg, childName='cacheGenerator') # Replace isDefault field value with False for invalid rows table.difference_update(invalidRows) for invalidRow in invalidRows: newRow = {} for field, value in invalidRow.items(): newRow[field] = False if field == 'isDefault' else value table.add(frozendict(newRow))
def _multiple_default_effects(dte_rows): """Check that each type has one default effect maximum.""" # Set with IDs of item types, which have default effect defeff_type_ids = set() invalid_rows = set() for row in sorted(dte_rows, key=lambda r: r['table_pos']): is_default = row.get('isDefault') # We're interested only in default effects if not is_default: continue type_id = row['typeID'] # If we already saw default effect for given type ID, invalidate # current row if type_id in defeff_type_ids: invalid_rows.add(row) else: defeff_type_ids.add(type_id) if invalid_rows: msg = ( 'data contains {} excessive default effects, ' 'marking them as non-default' ).format(len(invalid_rows)) logger.warning(msg) # Replace isDefault field value with False for invalid rows dte_rows.difference_update(invalid_rows) for invalid_row in invalid_rows: new_row = {} for field, value in invalid_row.items(): new_row[field] = False if field == 'isDefault' else value dte_rows.add(frozendict(new_row))
def _freeze_data(cls, data): if isinstance(data, dict): return frozendict({ cls._freeze_data(k): cls._freeze_data(v) for k, v in data.items()}) if isinstance(data, list): return tuple([cls._freeze_data(d) for d in data]) if isinstance(data, set): return frozenset([cls._freeze_data(d) for d in data]) return data
def _freezeModifier(self, modifier): """ Converts modifier into frozendict with its keys and values assigned according to modifier's ones. """ # Fields which we need to dump into row fields = ('state', 'context', 'sourceAttributeId', 'operator', 'targetAttributeId', 'location', 'filterType', 'filterValue') modifierRow = {} for field in fields: modifierRow[field] = getattr(modifier, field) frozenRow = frozendict(modifierRow) return frozenRow
def _move_attrs(data): """Normalize attribute value definitions. Some of item type attributes are defined in evetypes table. We do not need them there, for data consistency it's worth to move them to dgmtypeattribs table, where the rest of attributes are defined. Args: data: Dictionary in {table name: {table, rows}} format. """ attr_map = { 'radius': AttrId.radius, 'mass': AttrId.mass, 'volume': AttrId.volume, 'capacity': AttrId.capacity } attr_ids = tuple(attr_map.values()) # Here we will store pairs (typeID, attrID) already defined in # dgmtypeattribs defined_pairs = set() dgmtypeattribs = data['dgmtypeattribs'] for row in dgmtypeattribs: if row['attributeID'] not in attr_ids: continue defined_pairs.add((row['typeID'], row['attributeID'])) attrs_skipped = 0 for row in data['evetypes']: type_id = row['typeID'] for field, value in row.items(): if field in attr_map: # If row didn't have such attribute defined, skip it if value is None: continue # If such attribute already exists in dgmtypeattribs, do not # modify it - values from dgmtypeattribs table have priority attr_id = attr_map[field] if (type_id, attr_id) in defined_pairs: attrs_skipped += 1 continue # Generate row and add it to proper attribute table dgmtypeattribs.add( frozendict({ 'typeID': type_id, 'attributeID': attr_id, 'value': value })) if attrs_skipped: msg = ('{} built-in attributes already have had value ' 'in dgmtypeattribs and were skipped').format(attrs_skipped) logger.warning(msg)
def _move_attrs(data): """Normalize attribute value definitions. Some of item type attributes are defined in evetypes table. We do not need them there, for data consistency it's worth to move them to dgmtypeattribs table, where the rest of attributes are defined. Args: data: Dictionary in {table name: {table, rows}} format. """ attr_map = { 'radius': AttrId.radius, 'mass': AttrId.mass, 'volume': AttrId.volume, 'capacity': AttrId.capacity} attr_ids = tuple(attr_map.values()) # Here we will store pairs (typeID, attrID) already defined in # dgmtypeattribs defined_pairs = set() dgmtypeattribs = data['dgmtypeattribs'] for row in dgmtypeattribs: if row['attributeID'] not in attr_ids: continue defined_pairs.add((row['typeID'], row['attributeID'])) attrs_skipped = 0 for row in data['evetypes']: type_id = row['typeID'] for field, value in row.items(): if field in attr_map: # If row didn't have such attribute defined, skip it if value is None: continue # If such attribute already exists in dgmtypeattribs, do not # modify it - values from dgmtypeattribs table have priority attr_id = attr_map[field] if (type_id, attr_id) in defined_pairs: attrs_skipped += 1 continue # Generate row and add it to proper attribute table dgmtypeattribs.add(frozendict({ 'typeID': type_id, 'attributeID': attr_id, 'value': value})) if attrs_skipped: msg = ( '{} built-in attributes already have had value ' 'in dgmtypeattribs and were skipped' ).format(attrs_skipped) logger.warning(msg)
def run(self, dataHandler): """ Generate cache out of passed data. Positional arguments: dataHandler - data handler to use for getting data Return value: Dictionary in {entity type: [{field name: field value}] format """ # Put all the data we need into single dictionary # Format, as usual, {table name: table}, where table # is set of rows, which are represented by frozendicts # {fieldName: fieldValue}. Combination of sets and # frozendicts is used to speed up several stages of # the generator. data = {} tables = {'invtypes': dataHandler.getInvtypes, 'invgroups': dataHandler.getInvgroups, 'dgmattribs': dataHandler.getDgmattribs, 'dgmtypeattribs': dataHandler.getDgmtypeattribs, 'dgmeffects': dataHandler.getDgmeffects, 'dgmtypeeffects': dataHandler.getDgmtypeeffects, 'dgmexpressions': dataHandler.getDgmexpressions} for tablename, method in tables.items(): tablePos = 0 # For faster processing of various operations, # freeze table rows and put them into set table = set() for row in method(): # During further generator stages. some of rows # may fall in risk groups, where all rows but one # need to be removed. To deterministically remove rows # based on position in original data, write position # to each row row['tablePos'] = tablePos tablePos += 1 table.add(frozendict(row)) data[tablename] = table # Run pre-cleanup checks, as cleaning and further stages # rely on some assumptions about the data self._checker.preCleanup(data) # Also normalize the data to make data structure # more consistent, and thus easier to clean properly self._converter.normalize(data) # Clean our container out of unwanted data self._cleaner.clean(data) # Verify that our data is ready for conversion self._checker.preConvert(data) # Convert data into Eos-specific format. Here tables are # no longer represented by sets of frozendicts, but by # list of dicts data = self._converter.convert(data) return data
def _convert_expression_symbolic_references(data): """Convert all known symbolic references to int references. Some of entities in dgmexpressions table are defined not as IDs, but as symbolic references. CCP most likely has these symbolic names defined in their code, thus we have to hardcode it too. Args: data: Dictionary in {table name: {table, rows}} format. """ dgmexps = data['dgmexpressions'] # Replacement specification # Format: # ( # ( # operator, # column name for entity ID, # {replacement: map}, # (ignored names, ...) # ), # ... # ) repl_spec = ((OperandId.def_attr, 'expressionAttributeID', {}, ('shieldDamage', )), (OperandId.def_grp, 'expressionGroupID', { 'EnergyWeapon': TypeGroupId.energy_weapon, 'HybridWeapon': TypeGroupId.hydrid_weapon, 'MiningLaser': TypeGroupId.mining_laser, 'ProjectileWeapon': TypeGroupId.projectile_weapon }, ('Structure', 'PowerCore', ' None')), (OperandId.def_type, 'expressionTypeID', {}, ('Acceration Control', ))) for operand, id_col_name, repls, ignored_names in repl_spec: used_repls = set() unknown_names = set() # We're modifying only rows with specific operands for exp_row in dgmexps: if exp_row['operandID'] != operand: continue exp_entity_id = exp_row[id_col_name] # If entity is already referenced via ID, nothing to do here if exp_entity_id is not None: continue symbolic_entity_name = exp_row['expressionValue'] # Skip names we've set to ignore explicitly if symbolic_entity_name in ignored_names: continue # Do replacements if they're known to us if symbolic_entity_name in repls: # As rows are frozen dicts, we compose new mutable dict, # update data there, freeze it, and do actual replacement new_exp_row = {} new_exp_row.update(exp_row) new_exp_row['expressionValue'] = None new_exp_row[id_col_name] = repls[symbolic_entity_name] new_exp_row = frozendict(new_exp_row) dgmexps.remove(exp_row) dgmexps.add(new_exp_row) used_repls.add(symbolic_entity_name) continue # If we do not know it, add to special container which we will # use later unknown_names.add(symbolic_entity_name) # Report results to log, it will help to indicate when CCP finally # stops using literal references, and we can get rid of this # conversion, or add some new unused_repls = set(repls).difference(used_repls) if unused_repls: unused_repls = sorted(unused_repls) unused_line = ', '.join('"{}"'.format(r) for r in unused_repls) msg = '{} replacements for {} were not used: {}'.format( len(unused_repls), id_col_name, unused_line) logger.warning(msg) if unknown_names: unknown_names = sorted(unknown_names) unknown_line = ', '.join('"{}"'.format(n) for n in unknown_names) msg = ('unable to convert {} literal references to {}: {}' ).format(len(unknown_names), id_col_name, unknown_line) logger.warning(msg)
def run(data_handler): """Run eve object building process. Use data provided by passed cache handler to compose various objects with the help of which eos will oeprate. Args: data_handler: Data handler instance, which should provide access to raw eve data. Returns: 3 iterables, which contain types, attributes and effects. """ # Put all the data we need into single dictionary Format, as usual, # {table name: table}, where table is set of rows, which are # represented by frozendicts {fieldName: fieldValue}. Combination of # sets and frozendicts is used to speed up several stages of the # builder. data = {} getter_map = { 'evetypes': data_handler.get_evetypes, 'evegroups': data_handler.get_evegroups, 'dgmattribs': data_handler.get_dgmattribs, 'dgmtypeattribs': data_handler.get_dgmtypeattribs, 'dgmeffects': data_handler.get_dgmeffects, 'dgmtypeeffects': data_handler.get_dgmtypeeffects, 'dgmexpressions': data_handler.get_dgmexpressions, 'typefighterabils': data_handler.get_typefighterabils } for table_name, getter in getter_map.items(): table_pos = 0 table = set() for row in getter(): # During further builder stages. some of rows may fall in risk # groups, where all rows but one need to be removed. To # deterministically remove rows based on position in original # data, write position to each row row['table_pos'] = table_pos table_pos += 1 table.add(frozendict(row)) data[table_name] = table # Run pre-cleanup checks, as cleanup stage and further stages rely on # some assumptions about the data ValidatorPreClean.run(data) # Normalize the data to make data structure more consistent, making it # easier to clean properly Normalizer.run(data) # Remove unwanted data Cleaner().clean(data) # Verify that our data is ready for conversion ValidatorPreConv.run(data) # Convert data into Eos-specific objects types, attrs, effects = Converter.run(data) return types, attrs, effects
def _convert_expression_symbolic_references(data): """Convert all known symbolic references to int references. Some of entities in dgmexpressions table are defined not as IDs, but as symbolic references. CCP most likely has these symbolic names defined in their code, thus we have to hardcode it too. Args: data: Dictionary in {table name: {table, rows}} format. """ dgmexps = data['dgmexpressions'] # Replacement specification # Format: # ( # ( # operator, # column name for entity ID, # {replacement: map}, # (ignored names, ...) # ), # ... # ) repl_spec = ( ( OperandId.def_attr, 'expressionAttributeID', {}, ('shieldDamage',)), ( OperandId.def_grp, 'expressionGroupID', { 'EnergyWeapon': TypeGroupId.energy_weapon, 'HybridWeapon': TypeGroupId.hydrid_weapon, 'MiningLaser': TypeGroupId.mining_laser, 'ProjectileWeapon': TypeGroupId.projectile_weapon}, ('Structure', 'PowerCore', ' None')), ( OperandId.def_type, 'expressionTypeID', {}, ('Acceration Control',))) for operand, id_col_name, repls, ignored_names in repl_spec: used_repls = set() unknown_names = set() # We're modifying only rows with specific operands for exp_row in dgmexps: if exp_row['operandID'] != operand: continue exp_entity_id = exp_row[id_col_name] # If entity is already referenced via ID, nothing to do here if exp_entity_id is not None: continue symbolic_entity_name = exp_row['expressionValue'] # Skip names we've set to ignore explicitly if symbolic_entity_name in ignored_names: continue # Do replacements if they're known to us if symbolic_entity_name in repls: # As rows are frozen dicts, we compose new mutable dict, # update data there, freeze it, and do actual replacement new_exp_row = {} new_exp_row.update(exp_row) new_exp_row['expressionValue'] = None new_exp_row[id_col_name] = repls[symbolic_entity_name] new_exp_row = frozendict(new_exp_row) dgmexps.remove(exp_row) dgmexps.add(new_exp_row) used_repls.add(symbolic_entity_name) continue # If we do not know it, add to special container which we will # use later unknown_names.add(symbolic_entity_name) # Report results to log, it will help to indicate when CCP finally # stops using literal references, and we can get rid of this # conversion, or add some new unused_repls = set(repls).difference(used_repls) if unused_repls: unused_repls = sorted(unused_repls) unused_line = ', '.join('"{}"'.format(r) for r in unused_repls) msg = '{} replacements for {} were not used: {}'.format( len(unused_repls), id_col_name, unused_line) logger.warning(msg) if unknown_names: unknown_names = sorted(unknown_names) unknown_line = ', '.join( '"{}"'.format(n) for n in unknown_names) msg = ( 'unable to convert {} literal references to {}: {}' ).format(len(unknown_names), id_col_name, unknown_line) logger.warning(msg)