def countStopTimesPerStop(menages, menages_label, nbLignes, stopsToCheck, name, newCounter): tStart = time.clock() menages_label.append('1kmCount'+name+'stops') menages_label.append('500mCount'+name+'stops') for c in newCounter: menages_label.append('1kmCount'+name+c) menages_label.append('500mCount'+name+c) i = 0 for menage in menages.values(): menage['1kmCount'+name+'stops'] = 0 menage['500mCount'+name+'stops'] = 0 for c in newCounter: menage['1kmCount'+name+c] = 0 menage['500mCount'+name+c] = 0 ptA = [menage['mtmx'], menage['mtmy']] for stop in stopsToCheck.values(): ptB = [stop['mtmx'], stop['mtmy']] distance = getDistance(ptA, ptB) if distance < 500: menage['500mCount'+name+'stops'] += 1 for c in newCounter: menage['500mCount'+name+c] += stop[c] if distance < 1000: menage['1kmCount'+name+'stops'] += 1 for c in newCounter: menage['1kmCount'+name+c] += stop[c] toolbox.progressBar(i, nbLignes) i += 1 toolbox.hideProgressBar() print(len(stopsToCheck), 'stops affectés aux', i, "ménages en", toolbox.tempsCalulString(tStart), "pour les stops", name)
def loadFile(fileName, nbLignes, mapping): tStart = time.clock() stops = {} labels = [] isGtfsStm = 'stm' in fileName or 'STM' in fileName or 'Stm' in fileName i = -1 for line in open(fileName, encoding="utf8"): # , encoding="utf8" ligne = line.replace("\n", "").split(",") # name;lon;lat;coordsLatLon;capacity;region if i < 0: labels = ligne i += 1 continue elif i >= 2000000: break # On veut que 500 tapIn mais ... on ne veut pas couper les tap in d'une carte ! ligneJson = OrderedDict() j = 0 for field in labels: ligneJson[field] = parseValue(mapping, field, ligne[j]) j += 1 ligneJson['countStopTimes'] = 0 # On initialise le countDeStopTimes pour chaque arret à 0 if isGtfsStm: ligneJson['countStopTimesSemaine'] = 0 ligneJson['countStopTimesSamedi'] = 0 ligneJson['countStopTimesDimanche'] = 0 stops[ligneJson['stop_id']] = ligneJson toolbox.progressBar(i, nbLignes) i += 1 toolbox.hideProgressBar() print(i+1, "stops lus en", toolbox.tempsCalulString(tStart), "pour", fileName) labels.append('countStopTimes') if isGtfsStm: labels.extend(['countStopTimesSemaine', 'countStopTimesSamedi', 'countStopTimesDimanche']) return stops, labels
def countStopTimesPerStop(fileName, nbLignes, mapping, stops): tStart = time.clock() labels = [] stopIdPos = 3 tripIdPos = 0 isGtfsStm = 'stm' in fileName or 'STM' in fileName or 'Stm' in fileName i = -1 for line in open(fileName, encoding="utf8"): # , encoding="utf8" ligne = line.replace("\n", "").split(",") # name;lon;lat;coordsLatLon;capacity;region if i < 0: labels = ligne stopIdPos = labels.index('stop_id') tripIdPos = labels.index('trip_id') i += 1 continue elif i >= 20000000: break # On veut que 500 tapIn mais ... on ne veut pas couper les tap in d'une carte ! try: stops[ligne[stopIdPos]]['countStopTimes'] += 1 if isGtfsStm: tripId = ligne[tripIdPos] if '_S_' in tripId: stops[ligne[stopIdPos]]['countStopTimesSemaine'] += 1 elif '_A_' in tripId: stops[ligne[stopIdPos]]['countStopTimesSamedi'] += 1 elif '_I_' in tripId: stops[ligne[stopIdPos]]['countStopTimesDimanche'] += 1 except Exception: print(ligne[stopIdPos], 'inconu dans les stops du GTFS O.o,', stopIdPos) toolbox.progressBar(i, nbLignes) i += 1 toolbox.hideProgressBar() print(i, "stopTimes lus en", toolbox.tempsCalulString(tStart), "et affectés aux stops pour", fileName)
def loadFile(fileName, nbLignes, idPrincipale, mapping, separator): tStart = time.clock() stops = {} labels = [] isGtfsStm = 'stm' in fileName or 'STM' in fileName or 'Stm' in fileName i = -1 for line in open(fileName, encoding="utf8"): # , encoding="utf8" ligne = line.replace("\n", "").split(separator) # name;lon;lat;coordsLatLon;capacity;region if i < 0: labels = ligne i += 1 continue elif i >= 2000000: break # On veut que 500 tapIn mais ... on ne veut pas couper les tap in d'une carte ! ligneJson = OrderedDict() j = 0 try: for field in labels: ligneJson[field] = parseValue(mapping, field, ligne[j]) j += 1 except Exception: print(labels) print(ligne) stops[ligneJson[idPrincipale]] = ligneJson toolbox.progressBar(i, nbLignes) i += 1 toolbox.hideProgressBar() print(i+1, "entrées lues en", toolbox.tempsCalulString(tStart), "pour", fileName) return stops, labels
def exportFileToES(index, docType, fileName, date): tStart = time.clock() batch = [] labels = [] counts = { "i": 0, "countAddedDocs": 0, } countAddedDocs = 0 i = 0 data = json.loads(open(fileName).readline()) # ['type', 'geometry', 'properties'] # geometry: {'type', 'coordinates'} # properties: ['n_coord', 'length', 'purpose', 'start', 'id_origine', # 'id', 'liste_segments_jsonb', 'stop'] for feature in data['features']: if date: feature["date"] = date else: feature["date"] = feature["properties"]["start"] try: feature["hour"] = int(feature["date"][-8:-6]) except Exception: print("buuug", feature["properties"]) english2French = { 'Commute': 'Domicile-travail', 'Errand': 'Courses', 'Exercise': 'Sport', 'Leisure': 'Loisirs', 'Other': 'Autre', 'Autres': 'Autre', 'School': 'École', 'Shopping': 'Magasinage', 'Work-Related': 'Travail', 'Work-related': 'Travail', 'Other': 'Autre', 'other': 'Autre' } feature['properties']['purpose'] = english2French.get(feature['properties']['purpose'], feature['properties']['purpose']) if len(feature["geometry"]["coordinates"]) >= 2: feature['properties']["startPoint"] = feature["geometry"]["coordinates"][0] feature['properties']["endPoint"] = feature["geometry"]["coordinates"][-1] else: print("Oh, baa ya qu'un point ou pas du tout") # print(feature["date"][-8:-3]) # break toolbox.progressBar(i, 15800) if i >= 100000: # On veut que 500 tapIn mais ... on ne veut pas couper les tap in # d'une carte ! break batchToElasticSearch(feature, batch, counts, es, index, docType, False) i += 1 if batch: # On regarde les tap In précédents pour des correspondances batchToElasticSearch(feature, batch, counts, es, index, docType, True) toolbox.hideProgressBar() print(counts["countAddedDocs"], "/", i + 1, "transactions envoyées à ElasticSearch en", toolbox.tempsCalulString(tStart), "pour", fileName)
def exportFileToES(index, docType, fileName, date): tStart = time.clock() batch = [] labels = [] counts = { "i": 0, "countAddedDocs": 0, } countAddedDocs = 0 i = -1 for line in open(fileName, encoding="utf8"): ligne = line.replace("\n", "").split(";") # name;lon;lat;coordsLatLon;capacity;region if i < 0: labels = ligne if date: labels.append('date') else: if docType == "bixi_OD": labels.append('OD_stationsIds') labels.append('date') labels.append('hour') labels.append('tempsTrajet') labels.append('Start Coords LngLat') labels.append('End Coords LngLat') i += 1 continue if date: ligne.append(date) else: if docType == "bixi_OD": # 0:StartDate, 1:StartStationNumber, 2:StartStation, 3:EndDate, 4:EndStationNumber, 5:EndStation, 6:AccountType, 7:MemberGender, 8:TotalDuration, 9:MemberLanguage ligne.append(ligne[1]+"-"+ligne[4]) ligne.append(ligne[0]) ligne.append(ligne[0][-5:-3]) try: tempsTrajet = ligne[8].split("m") tempsTrajet = tempsTrajet[0].split("h") if len(tempsTrajet) == 2: tempsTrajet = int(tempsTrajet[0])*60 + int(tempsTrajet[1]) else: tempsTrajet = int(tempsTrajet[0]) except Exception: print("buug", ligne[8]) ligne.append(tempsTrajet) try: ligne.append(bixi_stops[ligne[1]]) except Exception: print("OD", ligne[1], "->", ligne[4], "éliminée, station", ligne[1], "inexistante", "ligne:", ligne) continue try: ligne.append(bixi_stops[ligne[4]]) except Exception: print("OD", ligne[1], "->", ligne[4], "éliminée, station", ligne[4], "inexistante", "ligne:", ligne) continue toolbox.progressBar(i, 700000) if i >= 2000000: break # On veut que 500 tapIn mais ... on ne veut pas couper les tap in d'une carte ! batchToElasticSearch(ligne, labels, batch, counts, es, index, docType, False) i += 1 if batch: # On regarde les tap In précédents pour des correspondances batchToElasticSearch(ligne, labels, batch, counts, es, index, docType, True) toolbox.hideProgressBar() print(counts["countAddedDocs"], "/", i+1, "transactions envoyées à ElasticSearch en", toolbox.tempsCalulString(tStart), "pour", fileName)
def aggregateTowardsSridu(menages, personnes, nbLignes): tStart = time.clock() sridus = {} sridu_labels = list(getSridu('sridu', sridus).keys()) sridus = {} i = 0 for pers in personnes.values(): factPers = pers['facteurPersonne'] menage = menages[pers['idFeuillet']] sridu = getSridu(pers['SRIDU'], sridus) if not sridu['SRIDU']: print(pers) updateAvg(menage['1kmCountBIXIstops'], factPers, sridu['1kmCountBIXIstops']) updateAvg(menage['500mCountBIXIstops'], factPers, sridu['500mCountBIXIstops']) updateAvg(menage['1kmCountBIXIcapacity'], factPers, sridu['1kmCountBIXIcapacity']) updateAvg(menage['500mCountBIXIcapacity'], factPers, sridu['500mCountBIXIcapacity']) updateAvg(menage['1kmCountAMTstops'], factPers, sridu['1kmCountAMTstops']) updateAvg(menage['500mCountAMTstops'], factPers, sridu['500mCountAMTstops']) updateAvg(menage['1kmCountAMTcountStopTimes'], factPers, sridu['1kmCountAMTcountStopTimes']) updateAvg(menage['500mCountAMTcountStopTimes'], factPers, sridu['500mCountAMTcountStopTimes']) updateAvg(menage['1kmCountSTMstops'], factPers, sridu['1kmCountSTMstops']) updateAvg(menage['500mCountSTMstops'], factPers, sridu['500mCountSTMstops']) updateAvg(menage['1kmCountSTMcountStopTimes'], factPers, sridu['1kmCountSTMcountStopTimes']) updateAvg(menage['500mCountSTMcountStopTimes'], factPers, sridu['500mCountSTMcountStopTimes']) updateAvg(menage['1kmCountSTMcountStopTimesSemaine'], factPers, sridu['1kmCountSTMcountStopTimesSemaine']) updateAvg(menage['500mCountSTMcountStopTimesSemaine'], factPers, sridu['500mCountSTMcountStopTimesSemaine']) updateAvg(menage['1kmCountSTMcountStopTimesSamedi'], factPers, sridu['1kmCountSTMcountStopTimesSamedi']) updateAvg(menage['500mCountSTMcountStopTimesSamedi'], factPers, sridu['500mCountSTMcountStopTimesSamedi']) updateAvg(menage['1kmCountSTMcountStopTimesDimanche'], factPers, sridu['1kmCountSTMcountStopTimesDimanche']) updateAvg(menage['500mCountSTMcountStopTimesDimanche'], factPers, sridu['500mCountSTMcountStopTimesDimanche']) updateAvg(pers['age'], factPers, sridu['age']) if pers['permis'] < 5: valPermis = 0 if pers['permis'] >= 2 else 1 updateAvg(valPermis, factPers, sridu['permis']) sridu['pop'] += factPers if pers['sexe'] == 1: sridu['hommes'] += factPers elif pers['sexe'] == 2: sridu['femmes'] += factPers statutTitle = 'statut_'+str(pers['statut']) sridu[statutTitle] += factPers toolbox.progressBar(i, nbLignes) i += 1 for sridu in sridus.values(): sridu['1kmCountBIXIstops'] = round(sridu['1kmCountBIXIstops']['sommePonderee'] / sridu['1kmCountBIXIstops']['quotient'], 1) sridu['500mCountBIXIstops'] = round(sridu['500mCountBIXIstops']['sommePonderee'] / sridu['500mCountBIXIstops']['quotient'], 1) sridu['1kmCountBIXIcapacity'] = round(sridu['1kmCountBIXIcapacity']['sommePonderee'] / sridu['1kmCountBIXIcapacity']['quotient'], 1) sridu['500mCountBIXIcapacity'] = round(sridu['500mCountBIXIcapacity']['sommePonderee'] / sridu['500mCountBIXIcapacity']['quotient'], 1) sridu['1kmCountAMTstops'] = round(sridu['1kmCountAMTstops']['sommePonderee'] / sridu['1kmCountAMTstops']['quotient'], 1) sridu['500mCountAMTstops'] = round(sridu['500mCountAMTstops']['sommePonderee'] / sridu['500mCountAMTstops']['quotient'], 1) sridu['1kmCountAMTcountStopTimes'] = round(sridu['1kmCountAMTcountStopTimes']['sommePonderee'] / sridu['1kmCountAMTcountStopTimes']['quotient'], 1) sridu['500mCountAMTcountStopTimes'] = round(sridu['500mCountAMTcountStopTimes']['sommePonderee'] / sridu['500mCountAMTcountStopTimes']['quotient'], 1) sridu['1kmCountSTMstops'] = round(sridu['1kmCountSTMstops']['sommePonderee'] / sridu['1kmCountSTMstops']['quotient'], 1) sridu['500mCountSTMstops'] = round(sridu['500mCountSTMstops']['sommePonderee'] / sridu['500mCountSTMstops']['quotient'], 1) sridu['1kmCountSTMcountStopTimes'] = round(sridu['1kmCountSTMcountStopTimes']['sommePonderee'] / sridu['1kmCountSTMcountStopTimes']['quotient'], 1) sridu['500mCountSTMcountStopTimes'] = round(sridu['500mCountSTMcountStopTimes']['sommePonderee'] / sridu['500mCountSTMcountStopTimes']['quotient'], 1) sridu['1kmCountSTMcountStopTimesSemaine'] = round(sridu['1kmCountSTMcountStopTimesSemaine']['sommePonderee'] / sridu['1kmCountSTMcountStopTimesSemaine']['quotient'], 1) sridu['500mCountSTMcountStopTimesSemaine'] = round(sridu['500mCountSTMcountStopTimesSemaine']['sommePonderee'] / sridu['500mCountSTMcountStopTimesSemaine']['quotient'], 1) sridu['1kmCountSTMcountStopTimesSamedi'] = round(sridu['1kmCountSTMcountStopTimesSamedi']['sommePonderee'] / sridu['1kmCountSTMcountStopTimesSamedi']['quotient'], 1) sridu['500mCountSTMcountStopTimesSamedi'] = round(sridu['500mCountSTMcountStopTimesSamedi']['sommePonderee'] / sridu['500mCountSTMcountStopTimesSamedi']['quotient'], 1) sridu['1kmCountSTMcountStopTimesDimanche'] = round(sridu['1kmCountSTMcountStopTimesDimanche']['sommePonderee'] / sridu['1kmCountSTMcountStopTimesDimanche']['quotient'], 1) sridu['500mCountSTMcountStopTimesDimanche'] = round(sridu['500mCountSTMcountStopTimesDimanche']['sommePonderee'] / sridu['500mCountSTMcountStopTimesDimanche']['quotient'], 1) sridu['age'] = round(sridu['age']['sommePonderee'] / sridu['age']['quotient'], 1) sridu['permis'] = round(sridu['permis']['sommePonderee'] / sridu['permis']['quotient'], 3) toolbox.hideProgressBar() print(len(personnes), 'personnes affectés aux', len(sridus), "Secteurs de Rescencement en", toolbox.tempsCalulString(tStart), "s") return sridus, sridu_labels