def content(table, version, fin, user): sess = None try: sess = Session() running_name, finished_name = chellow.dloads.make_names( table + '_' + version + '_general_import.csv', user) f = open(running_name, mode='w', newline='') w = csv.writer(f, lineterminator='\n') reader = iter(csv.reader(fin)) next(reader) if table == 'Line_Loss_Factor_Class': VOLTAGE_LEVEL_CODES = set( [v.code for v in sess.query(VoltageLevel)]) DNO_MAP = dict( (dno.participant.code, dno) for dno in sess.query(Party). join(MarketRole).filter(MarketRole.code == 'R').options( joinedload(Party.participant))) for i, values in enumerate(reader): participant_code = values[0] # market_role_code = values[1] llfc_code = values[3].zfill(3) valid_from = parse_date(values[4]) description = values[5] is_import = values[6] in ('A', 'B') is_substation = any( p in description for p in ( '_SS', ' SS', ' S/S', '(S/S)', 'sub', 'Sub')) valid_to = parse_to_date(values[7]) voltage_level_code = 'LV' description_upper = description.upper() for vl_code in VOLTAGE_LEVEL_CODES: if vl_code in description_upper: voltage_level_code = vl_code break try: dno = DNO_MAP[participant_code] except KeyError: w.writerow( ( "# There is no DNO with participant code ", participant_code)) continue llfc = sess.query(Llfc).filter( Llfc.dno == dno, Llfc.code == llfc_code, Llfc.valid_from == valid_from).first() if llfc is None: w.writerow( ( 'insert', 'llfc', dno.dno_code, llfc_code, description, voltage_level_code, is_substation, is_import, hh_format(valid_from), hh_format(valid_to, ongoing_str=''))) elif any( ( description != llfc.description, voltage_level_code != llfc.voltage_level.code, is_substation != llfc.is_substation, is_import != llfc.is_import, valid_to != llfc.valid_to)): w.writerow( ( 'update', 'llfc', dno.dno_code, llfc.code, hh_format(llfc.valid_from), description, voltage_level_code, is_substation, is_import, hh_format(valid_to, ongoing_str=''))) elif table == 'Market_Participant': for i, values in enumerate(reader): participant_code = values[0] participant_name = values[1] participant = sess.query(Participant).filter( Participant.code == participant_code).first() if participant is None: w.writerow( ( 'insert', 'participant', participant_code, participant_name)) elif participant_name != participant.name: w.writerow( ( 'update', 'participant', participant_code, participant_name)) elif table == 'Market_Role': for i, values in enumerate(reader): role_code = values[0] role_description = values[1] role = sess.query(MarketRole).filter( MarketRole.code == role_code).first() if role is None: w.writerow( ( 'insert', 'market_role', role_code, role_description)) elif role_description != role.description: w.writerow( ( 'update', 'market_role', role_code, role_description)) elif table == 'Market_Participant_Role': for i, values in enumerate(reader): participant_code = values[0] market_role_code = values[1] valid_from = parse_date(values[2]) party = sess.query(Party).join(Participant). \ join(MarketRole).filter( Party.valid_from == valid_from, Participant.code == participant_code, MarketRole.code == market_role_code).first() valid_to = parse_to_date(values[3]) name = values[4] dno_code_str = values[14] dno_code = None if len(dno_code_str) == 0 else dno_code_str if dno_code == '99': continue if party is None: w.writerow( ( 'insert', 'party', market_role_code, participant_code, name, hh_format(valid_from), hh_format(valid_to, ongoing_str=''), dno_code_str)) elif any( ( name != party.name, dno_code != party.dno_code, valid_to != party.valid_to)): w.writerow( ( 'update', 'party', market_role_code, participant_code, name, hh_format(valid_from), hh_format(valid_to, ongoing_str=''), dno_code_str)) elif table == 'Meter_Timeswitch_Class': for i, values in enumerate(reader): code = values[0].zfill(3) valid_from = parse_date(values[1]) valid_to = parse_to_date(values[2]) description = values[3] is_common = values[4] == 'T' has_related_metering = values[5] == 'T' meter_type_code = values[6] meter_payment_type_code = values[7] has_comms = values[8] == 'T' is_hh = values[9] == 'H' tpr_count_str = values[10] tpr_count = 0 if tpr_count_str == '' else int(tpr_count_str) if is_common: mtc = sess.query(Mtc).filter( Mtc.dno == null(), Mtc.code == code, Mtc.valid_from == valid_from).first() if mtc is None: w.writerow( ( 'insert', 'mtc', '', code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, hh_format(valid_from), hh_format(valid_to, ongoing_str=''))) elif any( ( description != mtc.description, has_related_metering != mtc.has_related_metering, has_comms != mtc.has_comms, is_hh != mtc.is_hh, meter_type_code != mtc.meter_type.code, meter_payment_type_code != mtc.meter_payment_type.code, tpr_count != mtc.tpr_count, valid_to != mtc.valid_to)): w.writerow( ( 'update', 'mtc', '', mtc.code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, hh_format(mtc.valid_from), hh_format(valid_to, ongoing_str=''))) elif table == 'MTC_in_PES_Area': dnos = dict( (p.participant.code, (p.id, p.dno_code)) for p in sess.query( Party).join(Participant).join(MarketRole).filter( MarketRole.code == 'R').options( joinedload(Party.participant))) mtcs = dict( ((m.dno_id, m.code, m.valid_from), m) for m in sess.query(Mtc).options( joinedload(Mtc.meter_type), joinedload(Mtc.meter_payment_type)).all()) for i, values in enumerate(reader): code_str = values[0] if not Mtc.has_dno(code_str): continue code_int = int(code_str) code = code_str.zfill(3) participant_code = values[2] dno_id, dno_code = dnos[participant_code] valid_from = parse_date(values[3]) valid_to = parse_to_date(values[4]) description = values[5] meter_type_code = values[6] meter_payment_type_code = values[7] has_related_metering = code_int > 500 has_comms = values[8] == 'Y' is_hh = values[9] == 'H' tpr_count_str = values[10] tpr_count = 0 if tpr_count_str == '' else int(tpr_count_str) mtc = mtcs.get((dno_id, code, valid_from)) if mtc is None: w.writerow( ( 'insert', 'mtc', dno_code, code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, hh_format(valid_from), hh_format(valid_to, ongoing_str=''))) elif any( ( description != mtc.description, has_related_metering != mtc.has_related_metering, has_comms != mtc.has_comms, is_hh != mtc.is_hh, meter_type_code != mtc.meter_type.code, meter_payment_type_code != mtc.meter_payment_type.code, tpr_count != mtc.tpr_count, valid_to != mtc.valid_to)): w.writerow( ( 'update', 'mtc', mtc.dno.dno_code, mtc.code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, hh_format(mtc.valid_from), hh_format(valid_to, ongoing_str=''))) elif table == 'MTC_Meter_Type': for i, values in enumerate(reader): code = values[0] description = values[1] valid_from = parse_date(values[2]) valid_to = parse_to_date(values[3]) pt = sess.query(MeterType).filter( MeterType.code == code, MeterType.valid_from == valid_from).first() if pt is None: w.writerow( ( 'insert', 'meter_type', code, description, hh_format(valid_from), hh_format(valid_to, ongoing_str=''))) elif (description, valid_from, valid_to) != ( pt.description, pt.valid_from, pt.valid_to): w.writerow( ( 'update', 'meter_type', code, description, hh_format(valid_from), hh_format(valid_to))) else: raise Exception("The table " + table + " is not recognized.") except BaseException: w.writerow([traceback.format_exc()]) finally: if sess is not None: sess.close() if f is not None: f.close() os.rename(running_name, finished_name)
def _parse_MTC_in_PES_Area(sess, csv_reader): rows = [] dnos = dict( (p.participant.code, (p.id, p.dno_code)) for p in sess.query(Party).join(Participant).join(MarketRole).filter( MarketRole.code == "R").options(joinedload(Party.participant))) mtcs = dict( ((m.dno_id, m.code, m.valid_from), m) for m in sess.query(Mtc).options(joinedload( Mtc.meter_type), joinedload(Mtc.meter_payment_type)).all()) for values in csv_reader: code_str = values[0] if not Mtc.has_dno(code_str): continue code_int = int(code_str) code = code_str.zfill(3) participant_code = values[2] dno_id, dno_code = dnos[participant_code] valid_from = parse_date(values[3]) valid_to = parse_to_date(values[4]) description = values[5] meter_type_code = values[6] meter_payment_type_code = values[7] has_related_metering = code_int > 500 has_comms = values[8] == "Y" is_hh = values[9] == "H" tpr_count_str = values[10] tpr_count = 0 if tpr_count_str == "" else int(tpr_count_str) mtc = mtcs.get((dno_id, code, valid_from)) if mtc is None: row = ( "insert", "mtc", dno_code, code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, hh_format(valid_from), hh_format(valid_to, ongoing_str=""), ) rows.append(row) elif any(( description != mtc.description, has_related_metering != mtc.has_related_metering, has_comms != mtc.has_comms, is_hh != mtc.is_hh, meter_type_code != mtc.meter_type.code, meter_payment_type_code != mtc.meter_payment_type.code, tpr_count != mtc.tpr_count, valid_to != mtc.valid_to, )): row = ( "update", "mtc", mtc.dno.dno_code, mtc.code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, hh_format(mtc.valid_from), hh_format(valid_to, ongoing_str=""), ) rows.append(row) return rows
def _import_MTC_in_PES_Area(sess, csv_reader): dnos = dict( (p.participant.code, p) for p in sess.query(Party).join(Participant).join(MarketRole).filter( MarketRole.code == "R").options(joinedload(Party.participant))) mtcs = dict( ((m.dno_id, m.code, m.valid_from), m) for m in sess.query(Mtc).options(joinedload( Mtc.meter_type), joinedload(Mtc.meter_payment_type)).all()) meter_types = dict( (m.code, m) for m in sess.execute(select(MeterType)).scalars()) meter_payment_types = dict( (m.code, m) for m in sess.execute(select(MeterPaymentType)).scalars()) for values in csv_reader: code_str = values[0] if not Mtc.has_dno(code_str): continue code_int = int(code_str) code = code_str.zfill(3) participant_code = values[2] dno = dnos[participant_code] valid_from = parse_date(values[3]) valid_to = parse_to_date(values[4]) description = values[5] meter_type_code = values[6] meter_type = meter_types[meter_type_code] meter_payment_type_code = values[7] meter_payment_type = meter_payment_types[meter_payment_type_code] has_related_metering = code_int > 500 has_comms = values[8] == "Y" is_hh = values[9] == "H" tpr_count_str = values[10] tpr_count = 0 if tpr_count_str == "" else int(tpr_count_str) mtc = mtcs.get((dno.id, code, valid_from)) if mtc is None: Mtc.insert( sess, dno.dno_code, code, description, has_related_metering, has_comms, is_hh, meter_type, meter_payment_type, tpr_count, valid_from, valid_to, ) else: mtc.description = description mtc.has_related_metering = has_related_metering mtc.has_comms = has_comms mtc.is_hh = is_hh mtc.meter_type = meter_type mtc.meter_payment_type = meter_payment_type mtc.tpr_count = tpr_count mtc.valid_to = valid_to sess.flush()
def _import_Valid_MTC_LLFC_SSC_PC_Combination(sess, csv_reader): dnos = {} mtcs = {} llfcs = {} sscs = {} pcs = dict((v.code, v) for v in sess.execute(select(Pc)).scalars()) combos = dict(((v.mtc.id, v.llfc.id, v.ssc.id, v.pc.id, v.valid_from), v) for v in sess.execute(select(ValidMtcLlfcSscPc)).scalars()) for values in csv_reader: mtc_code = values[0].zfill(3) # Meter Timeswitch Class ID # Effective From Settlement Date (MTC) participant_code = values[2] # Market Participant ID mtc_from_str = values[3] # Effective From Settlement Date (MTCPA) mtc_from = parse_date(mtc_from_str) ssc_code = values[4] # Standard Settlement Configuration ID ssc_from_str = values[5] # Effective From Settlement date (VMTCSC) ssc_from = parse_date(ssc_from_str) llfc_code = values[6] # Line Loss Factor Class ID llfc_from_str = values[7] # Effective From Settlement Date (VMTCLSC) llfc_from = parse_date(llfc_from_str) pc_code = values[8].zfill(2) # Profile Class ID valid_from_str = values[9] # Effective From Settlement Date (VMTCLSPC) valid_from = parse_date(valid_from_str) valid_to_str = values[10] # Effective To Settlement Date (VMTCLSPC) valid_to = parse_date(valid_to_str) # Preserved Tariff Indicator try: dno = dnos[(participant_code, valid_from)] except KeyError: dno = dnos[(participant_code, valid_from)] = sess.execute( select(Party).join(Participant).join(MarketRole).where( Participant.code == participant_code, MarketRole.code == "R", Party.valid_from <= valid_from, or_(Party.valid_to == null(), Party.valid_to >= valid_from), )).scalar_one() mtc_dno_id = dno.id if Mtc.has_dno(mtc_code) else None try: mtc = mtcs[(mtc_dno_id, mtc_code, mtc_from)] except KeyError: mtc = mtcs[(mtc_dno_id, mtc_code, mtc_from)] = Mtc.get_by_code(sess, dno, mtc_code, mtc_from) try: llfc = llfcs[(dno.id, llfc_code, llfc_from)] except KeyError: llfc = llfcs[(dno.id, llfc_code, llfc_from)] = dno.get_llfc_by_code( sess, llfc_code, llfc_from) try: ssc = sscs[(ssc_code, ssc_from)] except KeyError: ssc = sscs[(ssc_code, ssc_from)] = Ssc.get_by_code(sess, ssc_code, ssc_from) pc = pcs[pc_code] combo = combos.get((mtc.id, llfc.id, ssc.id, pc.id, valid_from)) if combo is None: ValidMtcLlfcSscPc.insert( sess, mtc, llfc, ssc, pc, valid_from, valid_to, ) else: combo.valid_to = valid_to sess.flush()
def content(table, version, f, sess): reader = iter(csv.reader(f)) next(reader) if table == 'Line_Loss_Factor_Class': LLFC_MAP = dict( ((llfc.dno.participant.code, llfc.code), llfc) for llfc in sess.query(Llfc).join(Party).options( joinedload(Llfc.dno).joinedload('participant'))) VOLTAGE_LEVEL_CODES = set( [v.code for v in sess.query(VoltageLevel)]) DNO_MAP = dict( (dno.participant.code, dno) for dno in sess.query(Party). join(MarketRole).filter(MarketRole.code == 'R').options( joinedload(Party.participant))) for i, values in enumerate(reader): participant_code = values[0] # market_role_code = values[1] from_date_mpr = values[2] llfc_code_raw = values[3] # from_date_settlement = values[4] llfc_description = values[5] class_indicator = values[6] to_date_settlement = values[7] llfc_code = llfc_code_raw.zfill(3) llfc = LLFC_MAP.get((participant_code, llfc_code)) if llfc is None: try: dno = DNO_MAP[participant_code] except KeyError: yield ''.join( "# There is no DNO with participant code ", participant_code, ".\n") continue voltage_level_code = 'LV' llfc_description_upper = llfc_description.upper() for vl_code in VOLTAGE_LEVEL_CODES: if vl_code in llfc_description_upper: voltage_level_code = vl_code break is_substation = any( p in llfc_description for p in [ '_SS', ' SS', ' S/S', '(S/S)', 'sub', 'Sub']) is_import = not any( p in class_indicator for p in ['C', 'D']) yield ','.join( ( '"' + str(v) + '"' for v in ( 'insert', 'llfc', dno.dno_code, llfc_code, llfc_description, voltage_level_code, is_substation, is_import, to_iso(from_date_mpr), to_iso(to_date_settlement)))) + "\n" elif table == 'Market_Participant': for i, values in enumerate(reader): participant_code = values[0] participant_name = values[1] participant = sess.query(Participant).filter( Participant.code == participant_code).first() if participant is None: yield ','.join( ( '"' + str(v) + '"' for v in ( 'insert', 'participant', participant_code, participant_name))) + "\n" elif participant_name != participant.name: yield ','.join( ( '"' + str(v) + '"' for v in ( 'update', 'participant', participant_code, participant_name))) + "\n" elif table == 'Market_Role': for i, values in enumerate(reader): role_code = values[0] role_description = values[1] role = sess.query(MarketRole).filter( MarketRole.code == role_code).first() if role is None: yield ','.join( ( '"' + str(v) + '"' for v in ( 'insert', 'market_role', role_code, role_description))) + "\n" elif role_description != role.description: yield ','.join( ( '"' + str(v) + '"' for v in ( 'update', 'market_role', role_code, role_description))) + "\n" elif table == 'Market_Participant_Role': for i, values in enumerate(reader): participant_code = values[0] market_role_code = values[1] party = sess.query(Party).join(Participant). \ join(MarketRole).filter( Participant.code == participant_code, MarketRole.code == market_role_code).first() valid_from_str = values[2] valid_from = Datetime.strptime(valid_from_str, "%d/%m/%Y") valid_to_str = values[3] if valid_to_str == '': valid_to = None else: valid_to = Datetime.strptime(valid_to_str, "%d/%m/%Y") name = values[4] dno_code_str = values[14] if len(dno_code_str) == 0: dno_code = None else: dno_code = dno_code_str if party is None: yield ','.join( ( '"' + str(v) + '"' for v in ( 'insert', 'party', market_role_code, participant_code, name, hh_format(valid_from), '' if valid_to is None else hh_format(valid_to), dno_code_str))) + "\n" elif name != party.name or dno_code != party.dno_code: yield ','.join( ( '"' + str(v) + '"' for v in ( 'update', 'party', market_role_code, participant_code, name, hh_format(valid_from), '' if valid_to is None else hh_format(valid_to), dno_code_str))) + "\n" elif table == 'Meter_Timeswitch_Class': for i, values in enumerate(reader): code_str = values[0] code_int = int(code_str) if is_common_mtc(code_int): code = code_str.zfill(3) valid_from_str = values[1] valid_from = Datetime.strptime( valid_from_str, "%d/%m/%Y").replace(tzinfo=pytz.utc) valid_from_out = hh_format(valid_from) valid_to_str = values[2] if valid_to_str == '': valid_to = None valid_to_out = '' else: valid_to = Datetime.strptime( valid_to_str, "%d/%m/%Y").replace(tzinfo=pytz.utc) valid_to_out = hh_format(valid_to) description = values[3] # common_code_indicator = values[4] has_related_metering_str = values[5] has_related_metering = has_related_metering_str == 'T' meter_type_code = values[6] meter_payment_type_code = values[7] has_comms_str = values[8] has_comms = has_comms_str == 'T' is_hh_str = values[9] is_hh = is_hh_str == 'H' tpr_count_str = values[10] if tpr_count_str == '': tpr_count = 0 else: tpr_count = int(tpr_count_str) mtc = Mtc.find_by_code(sess, None, code) if mtc is None: yield ','.join( ( '"' + str(v) + '"' for v in ( 'insert', 'mtc', '', code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, valid_from_out, valid_to_out))) + "\n" elif ( description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, valid_from, valid_to) != ( mtc.description, mtc.has_related_metering, mtc.has_comms, mtc.is_hh, mtc.meter_type.code, mtc.meter_payment_type.code, mtc.tpr_count, mtc.valid_from, mtc.valid_to): yield ','.join( ( '"' + str(v) + '"' for v in ( 'update', 'mtc', '', code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, valid_from_out, valid_to_out))) + "\n" elif table == 'MTC_in_PES_Area': dnos = dict( (p.participant.code, (p.id, p.dno_code)) for p in sess.query( Party).join(Participant).join(MarketRole).filter( MarketRole.code == 'R').options( joinedload(Party.participant))) mtcs = dict( ((m.dno_id, m.code), m) for m in sess.query(Mtc).options( joinedload(Mtc.meter_type), joinedload(Mtc.meter_payment_type)).all()) for i, values in enumerate(reader): code_str = values[0] code_int = int(code_str) if not is_common_mtc(code_int): code = code_str.zfill(3) participant_code = values[2] dno_id, dno_code = dnos[participant_code] valid_from_str = values[3] valid_from = Datetime.strptime( valid_from_str, "%d/%m/%Y").replace(tzinfo=pytz.utc) valid_from_out = hh_format(valid_from) valid_to_str = values[4] if valid_to_str == '': valid_to = None valid_to_out = '' else: valid_to = Datetime.strptime( valid_to_str, "%d/%m/%Y").replace(tzinfo=pytz.utc) valid_to_out = hh_format(valid_to) description = values[5] meter_type_code = values[6] meter_payment_type_code = values[7] has_related_metering = code_int > 500 has_comms = values[8] == 'Y' is_hh = values[9] == 'H' tpr_count_str = values[10] if tpr_count_str == '': tpr_count = 0 else: tpr_count = int(tpr_count_str) mtc_dno_id = dno_id if Mtc.has_dno(code) else None mtc = mtcs.get((mtc_dno_id, code)) if mtc is None: yield ','.join( ( '"' + str(v) + '"' for v in ( 'insert', 'mtc', dno_code, code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, valid_from_out, valid_to_out))) + "\n" elif ( description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, valid_from, valid_to) != ( mtc.description, mtc.has_related_metering, mtc.has_comms, mtc.is_hh, mtc.meter_type.code, mtc.meter_payment_type.code, mtc.tpr_count, mtc.valid_from, mtc.valid_to): yield ','.join( ( '"' + str(v) + '"' for v in ( 'update', 'mtc', dno_code, code, description, has_related_metering, has_comms, is_hh, meter_type_code, meter_payment_type_code, tpr_count, valid_from_out, valid_to_out))) + "\n" elif table == 'MTC_Meter_Type': for i, values in enumerate(reader): code = values[0] description = values[1] valid_from_str = values[2] valid_from = Datetime.strptime( valid_from_str, "%d/%m/%Y").replace(tzinfo=pytz.utc) valid_from_out = hh_format(valid_from) valid_to_str = values[3] if valid_to_str == '': valid_to = None valid_to_out = '' else: valid_to = Datetime.strptime( valid_to_str, "%d/%m/%Y").replace(tzinfo=pytz.utc) valid_to_out = hh_format(valid_to) pt = sess.query(MeterType).filter( MeterType.code == code).first() if pt is None: yield ','.join( ( '"' + str(v) + '"' for v in ( 'insert', 'meter_type', code, description, valid_from_out, valid_to_out))) + "\n" elif (description, valid_from, valid_to) != ( pt.description, pt.valid_from, pt.valid_to): yield ','.join( ( '"' + str(v) + '"' for v in ( 'update', 'meter_type', code, description, valid_from_out, valid_to_out))) + "\n" else: raise Exception("The table " + table + " is not recognized.")