def countStopTimesPerStop(menages, menages_label, nbLignes, stopsToCheck, name, newCounter):
    tStart = time.clock()
    menages_label.append('1kmCount'+name+'stops')
    menages_label.append('500mCount'+name+'stops')
    for c in newCounter:
        menages_label.append('1kmCount'+name+c)
        menages_label.append('500mCount'+name+c)

    i = 0
    for menage in menages.values():
        menage['1kmCount'+name+'stops'] = 0
        menage['500mCount'+name+'stops'] = 0
        for c in newCounter:
            menage['1kmCount'+name+c] = 0
            menage['500mCount'+name+c] = 0
        ptA = [menage['mtmx'], menage['mtmy']]
        for stop in stopsToCheck.values():
            ptB = [stop['mtmx'], stop['mtmy']]
            distance = getDistance(ptA, ptB)
            if distance < 500:
                menage['500mCount'+name+'stops'] += 1
                for c in newCounter:
                    menage['500mCount'+name+c] += stop[c]
            if distance < 1000:
                menage['1kmCount'+name+'stops'] += 1
                for c in newCounter:
                    menage['1kmCount'+name+c] += stop[c]

        toolbox.progressBar(i, nbLignes)
        i += 1

    toolbox.hideProgressBar()
    print(len(stopsToCheck), 'stops affectés aux', i, "ménages en", toolbox.tempsCalulString(tStart), "pour les stops", name)
def loadFile(fileName, nbLignes, mapping):
    tStart = time.clock()
    stops = {}
    labels = []

    isGtfsStm = 'stm' in fileName or 'STM' in fileName or 'Stm' in fileName

    i = -1
    for line in open(fileName, encoding="utf8"):  # , encoding="utf8"
        ligne = line.replace("\n", "").split(",")
        # name;lon;lat;coordsLatLon;capacity;region
        if i < 0:
            labels = ligne
            i += 1
            continue
        elif i >= 2000000:
            break  # On veut que 500 tapIn mais ... on ne veut pas couper les tap in d'une carte !

        ligneJson = OrderedDict()
        j = 0
        for field in labels:
            ligneJson[field] = parseValue(mapping, field, ligne[j])
            j += 1

        ligneJson['countStopTimes'] = 0  # On initialise le countDeStopTimes pour chaque arret à 0
        if isGtfsStm:
            ligneJson['countStopTimesSemaine'] = 0
            ligneJson['countStopTimesSamedi'] = 0
            ligneJson['countStopTimesDimanche'] = 0

        stops[ligneJson['stop_id']] = ligneJson

        toolbox.progressBar(i, nbLignes)
        i += 1

    toolbox.hideProgressBar()
    print(i+1, "stops lus en", toolbox.tempsCalulString(tStart), "pour", fileName)

    labels.append('countStopTimes')
    if isGtfsStm:
        labels.extend(['countStopTimesSemaine', 'countStopTimesSamedi', 'countStopTimesDimanche'])

    return stops, labels
def countStopTimesPerStop(fileName, nbLignes, mapping, stops):
    tStart = time.clock()
    labels = []

    stopIdPos = 3
    tripIdPos = 0
    isGtfsStm = 'stm' in fileName or 'STM' in fileName or 'Stm' in fileName

    i = -1
    for line in open(fileName, encoding="utf8"):  # , encoding="utf8"
        ligne = line.replace("\n", "").split(",")
        # name;lon;lat;coordsLatLon;capacity;region
        if i < 0:
            labels = ligne
            stopIdPos = labels.index('stop_id')
            tripIdPos = labels.index('trip_id')
            i += 1
            continue
        elif i >= 20000000:
            break  # On veut que 500 tapIn mais ... on ne veut pas couper les tap in d'une carte !

        try:
            stops[ligne[stopIdPos]]['countStopTimes'] += 1
            if isGtfsStm:
                tripId = ligne[tripIdPos]
                if '_S_' in tripId:
                    stops[ligne[stopIdPos]]['countStopTimesSemaine'] += 1
                elif '_A_' in tripId:
                    stops[ligne[stopIdPos]]['countStopTimesSamedi'] += 1
                elif '_I_' in tripId:
                    stops[ligne[stopIdPos]]['countStopTimesDimanche'] += 1
        except Exception:
            print(ligne[stopIdPos], 'inconu dans les stops du GTFS O.o,', stopIdPos)

        toolbox.progressBar(i, nbLignes)
        i += 1

    toolbox.hideProgressBar()
    print(i, "stopTimes lus en", toolbox.tempsCalulString(tStart), "et affectés aux stops pour", fileName)
def loadFile(fileName, nbLignes, idPrincipale, mapping, separator):
    tStart = time.clock()
    stops = {}
    labels = []

    isGtfsStm = 'stm' in fileName or 'STM' in fileName or 'Stm' in fileName

    i = -1
    for line in open(fileName, encoding="utf8"):  # , encoding="utf8"
        ligne = line.replace("\n", "").split(separator)
        # name;lon;lat;coordsLatLon;capacity;region
        if i < 0:
            labels = ligne
            i += 1
            continue
        elif i >= 2000000:
            break  # On veut que 500 tapIn mais ... on ne veut pas couper les tap in d'une carte !

        ligneJson = OrderedDict()
        j = 0
        try:
            for field in labels:
                ligneJson[field] = parseValue(mapping, field, ligne[j])
                j += 1
        except Exception:
            print(labels)
            print(ligne)

        stops[ligneJson[idPrincipale]] = ligneJson

        toolbox.progressBar(i, nbLignes)
        i += 1

    toolbox.hideProgressBar()
    print(i+1, "entrées lues en", toolbox.tempsCalulString(tStart), "pour", fileName)

    return stops, labels
Esempio n. 5
0
def exportFileToES(index, docType, fileName, date):
    tStart = time.clock()
    batch = []
    labels = []

    counts = {
        "i": 0,
        "countAddedDocs": 0,
    }

    countAddedDocs = 0
    i = 0

    data = json.loads(open(fileName).readline())
    # ['type', 'geometry', 'properties']
    # geometry: {'type', 'coordinates'}
    # properties: ['n_coord', 'length', 'purpose', 'start', 'id_origine',
    # 'id', 'liste_segments_jsonb', 'stop']

    for feature in data['features']:
        if date:
            feature["date"] = date
        else:
            feature["date"] = feature["properties"]["start"]
            try:
                feature["hour"] = int(feature["date"][-8:-6])
            except Exception:
                print("buuug", feature["properties"])

            english2French = {
                'Commute': 'Domicile-travail',
                'Errand': 'Courses',
                'Exercise': 'Sport',
                'Leisure': 'Loisirs',
                'Other': 'Autre',
                'Autres': 'Autre',
                'School': 'École',
                'Shopping': 'Magasinage',
                'Work-Related': 'Travail',
                'Work-related': 'Travail',
                'Other': 'Autre',
                'other': 'Autre'
            }
            feature['properties']['purpose'] = english2French.get(feature['properties']['purpose'], feature['properties']['purpose'])

            if len(feature["geometry"]["coordinates"]) >= 2:
                feature['properties']["startPoint"] = feature["geometry"]["coordinates"][0]
                feature['properties']["endPoint"] = feature["geometry"]["coordinates"][-1]
            else:
                print("Oh, baa ya qu'un point ou pas du tout")

            # print(feature["date"][-8:-3])
            # break

        toolbox.progressBar(i, 15800)

        if i >= 100000:
            # On veut que 500 tapIn mais ... on ne veut pas couper les tap in
            # d'une carte !
            break

        batchToElasticSearch(feature, batch, counts, es, index, docType, False)
        i += 1

    if batch:  # On regarde les tap In précédents pour des correspondances
        batchToElasticSearch(feature, batch, counts, es, index, docType, True)

    toolbox.hideProgressBar()
    print(counts["countAddedDocs"], "/", i + 1, "transactions envoyées à ElasticSearch en",
          toolbox.tempsCalulString(tStart), "pour", fileName)
def exportFileToES(index, docType, fileName, date):
    tStart = time.clock()
    batch = []
    labels = []

    counts = {
        "i": 0,
        "countAddedDocs": 0,
    }

    countAddedDocs = 0
    i = -1

    for line in open(fileName, encoding="utf8"):
        ligne = line.replace("\n", "").split(";")
        # name;lon;lat;coordsLatLon;capacity;region
        if i < 0:
            labels = ligne
            if date:
                labels.append('date')
            else:
                if docType == "bixi_OD":
                    labels.append('OD_stationsIds')
                    labels.append('date')
                    labels.append('hour')
                    labels.append('tempsTrajet')
                    labels.append('Start Coords LngLat')
                    labels.append('End Coords LngLat')
            i += 1
            continue
        if date:
            ligne.append(date)
        else:
            if docType == "bixi_OD":
                # 0:StartDate, 1:StartStationNumber, 2:StartStation, 3:EndDate, 4:EndStationNumber, 5:EndStation, 6:AccountType, 7:MemberGender, 8:TotalDuration, 9:MemberLanguage
                ligne.append(ligne[1]+"-"+ligne[4])
                ligne.append(ligne[0])
                ligne.append(ligne[0][-5:-3])
                try:
                    tempsTrajet = ligne[8].split("m")
                    tempsTrajet = tempsTrajet[0].split("h")
                    if len(tempsTrajet) == 2:
                        tempsTrajet = int(tempsTrajet[0])*60 + int(tempsTrajet[1])
                    else:
                        tempsTrajet = int(tempsTrajet[0])
                except Exception:
                    print("buug", ligne[8])
                ligne.append(tempsTrajet)
                try:
                    ligne.append(bixi_stops[ligne[1]])
                except Exception:
                    print("OD", ligne[1], "->", ligne[4], "éliminée, station", ligne[1], "inexistante", "ligne:", ligne)
                    continue
                try:
                    ligne.append(bixi_stops[ligne[4]])
                except Exception:
                    print("OD", ligne[1], "->", ligne[4], "éliminée, station", ligne[4], "inexistante", "ligne:", ligne)
                    continue
        toolbox.progressBar(i, 700000)

        if i >= 2000000:
            break  # On veut que 500 tapIn mais ... on ne veut pas couper les tap in d'une carte !

        batchToElasticSearch(ligne, labels, batch, counts, es, index, docType, False)
        i += 1

    if batch:  # On regarde les tap In précédents pour des correspondances
        batchToElasticSearch(ligne, labels, batch, counts, es, index, docType, True)

    toolbox.hideProgressBar()
    print(counts["countAddedDocs"], "/", i+1, "transactions envoyées à ElasticSearch en", toolbox.tempsCalulString(tStart), "pour", fileName)
def aggregateTowardsSridu(menages, personnes, nbLignes):
    tStart = time.clock()
    sridus = {}
    sridu_labels = list(getSridu('sridu', sridus).keys())
    sridus = {}

    i = 0
    for pers in personnes.values():
        factPers = pers['facteurPersonne']
        menage = menages[pers['idFeuillet']]
        sridu = getSridu(pers['SRIDU'], sridus)
        if not sridu['SRIDU']:
            print(pers)

        updateAvg(menage['1kmCountBIXIstops'], factPers, sridu['1kmCountBIXIstops'])
        updateAvg(menage['500mCountBIXIstops'], factPers, sridu['500mCountBIXIstops'])
        updateAvg(menage['1kmCountBIXIcapacity'], factPers, sridu['1kmCountBIXIcapacity'])
        updateAvg(menage['500mCountBIXIcapacity'], factPers, sridu['500mCountBIXIcapacity'])
        updateAvg(menage['1kmCountAMTstops'], factPers, sridu['1kmCountAMTstops'])
        updateAvg(menage['500mCountAMTstops'], factPers, sridu['500mCountAMTstops'])
        updateAvg(menage['1kmCountAMTcountStopTimes'], factPers, sridu['1kmCountAMTcountStopTimes'])
        updateAvg(menage['500mCountAMTcountStopTimes'], factPers, sridu['500mCountAMTcountStopTimes'])
        updateAvg(menage['1kmCountSTMstops'], factPers, sridu['1kmCountSTMstops'])
        updateAvg(menage['500mCountSTMstops'], factPers, sridu['500mCountSTMstops'])
        updateAvg(menage['1kmCountSTMcountStopTimes'], factPers, sridu['1kmCountSTMcountStopTimes'])
        updateAvg(menage['500mCountSTMcountStopTimes'], factPers, sridu['500mCountSTMcountStopTimes'])
        updateAvg(menage['1kmCountSTMcountStopTimesSemaine'], factPers, sridu['1kmCountSTMcountStopTimesSemaine'])
        updateAvg(menage['500mCountSTMcountStopTimesSemaine'], factPers, sridu['500mCountSTMcountStopTimesSemaine'])
        updateAvg(menage['1kmCountSTMcountStopTimesSamedi'], factPers, sridu['1kmCountSTMcountStopTimesSamedi'])
        updateAvg(menage['500mCountSTMcountStopTimesSamedi'], factPers, sridu['500mCountSTMcountStopTimesSamedi'])
        updateAvg(menage['1kmCountSTMcountStopTimesDimanche'], factPers, sridu['1kmCountSTMcountStopTimesDimanche'])
        updateAvg(menage['500mCountSTMcountStopTimesDimanche'], factPers, sridu['500mCountSTMcountStopTimesDimanche'])
        updateAvg(pers['age'], factPers, sridu['age'])
        if pers['permis'] < 5:
            valPermis = 0 if pers['permis'] >= 2 else 1
            updateAvg(valPermis, factPers, sridu['permis'])
        sridu['pop'] += factPers
        if pers['sexe'] == 1:
            sridu['hommes'] += factPers
        elif pers['sexe'] == 2:
            sridu['femmes'] += factPers
        statutTitle = 'statut_'+str(pers['statut'])
        sridu[statutTitle] += factPers

        toolbox.progressBar(i, nbLignes)
        i += 1

    for sridu in sridus.values():
        sridu['1kmCountBIXIstops'] = round(sridu['1kmCountBIXIstops']['sommePonderee'] / sridu['1kmCountBIXIstops']['quotient'], 1)
        sridu['500mCountBIXIstops'] = round(sridu['500mCountBIXIstops']['sommePonderee'] / sridu['500mCountBIXIstops']['quotient'], 1)
        sridu['1kmCountBIXIcapacity'] = round(sridu['1kmCountBIXIcapacity']['sommePonderee'] / sridu['1kmCountBIXIcapacity']['quotient'], 1)
        sridu['500mCountBIXIcapacity'] = round(sridu['500mCountBIXIcapacity']['sommePonderee'] / sridu['500mCountBIXIcapacity']['quotient'], 1)
        sridu['1kmCountAMTstops'] = round(sridu['1kmCountAMTstops']['sommePonderee'] / sridu['1kmCountAMTstops']['quotient'], 1)
        sridu['500mCountAMTstops'] = round(sridu['500mCountAMTstops']['sommePonderee'] / sridu['500mCountAMTstops']['quotient'], 1)
        sridu['1kmCountAMTcountStopTimes'] = round(sridu['1kmCountAMTcountStopTimes']['sommePonderee'] / sridu['1kmCountAMTcountStopTimes']['quotient'], 1)
        sridu['500mCountAMTcountStopTimes'] = round(sridu['500mCountAMTcountStopTimes']['sommePonderee'] / sridu['500mCountAMTcountStopTimes']['quotient'], 1)
        sridu['1kmCountSTMstops'] = round(sridu['1kmCountSTMstops']['sommePonderee'] / sridu['1kmCountSTMstops']['quotient'], 1)
        sridu['500mCountSTMstops'] = round(sridu['500mCountSTMstops']['sommePonderee'] / sridu['500mCountSTMstops']['quotient'], 1)
        sridu['1kmCountSTMcountStopTimes'] = round(sridu['1kmCountSTMcountStopTimes']['sommePonderee'] / sridu['1kmCountSTMcountStopTimes']['quotient'], 1)
        sridu['500mCountSTMcountStopTimes'] = round(sridu['500mCountSTMcountStopTimes']['sommePonderee'] / sridu['500mCountSTMcountStopTimes']['quotient'], 1)
        sridu['1kmCountSTMcountStopTimesSemaine'] = round(sridu['1kmCountSTMcountStopTimesSemaine']['sommePonderee'] / sridu['1kmCountSTMcountStopTimesSemaine']['quotient'], 1)
        sridu['500mCountSTMcountStopTimesSemaine'] = round(sridu['500mCountSTMcountStopTimesSemaine']['sommePonderee'] / sridu['500mCountSTMcountStopTimesSemaine']['quotient'], 1)
        sridu['1kmCountSTMcountStopTimesSamedi'] = round(sridu['1kmCountSTMcountStopTimesSamedi']['sommePonderee'] / sridu['1kmCountSTMcountStopTimesSamedi']['quotient'], 1)
        sridu['500mCountSTMcountStopTimesSamedi'] = round(sridu['500mCountSTMcountStopTimesSamedi']['sommePonderee'] / sridu['500mCountSTMcountStopTimesSamedi']['quotient'], 1)
        sridu['1kmCountSTMcountStopTimesDimanche'] = round(sridu['1kmCountSTMcountStopTimesDimanche']['sommePonderee'] / sridu['1kmCountSTMcountStopTimesDimanche']['quotient'], 1)
        sridu['500mCountSTMcountStopTimesDimanche'] = round(sridu['500mCountSTMcountStopTimesDimanche']['sommePonderee'] / sridu['500mCountSTMcountStopTimesDimanche']['quotient'], 1)
        sridu['age'] = round(sridu['age']['sommePonderee'] / sridu['age']['quotient'], 1)
        sridu['permis'] = round(sridu['permis']['sommePonderee'] / sridu['permis']['quotient'], 3)

    toolbox.hideProgressBar()
    print(len(personnes), 'personnes affectés aux', len(sridus), "Secteurs de Rescencement en", toolbox.tempsCalulString(tStart), "s")

    return sridus, sridu_labels