def loadVideos(catalog): """ Carga los videos del archivo. Por cada video se toman los datos necesarios: video id, trending date, category id, views, nombre del canal, país, nombre del video, likes, dislikes, fecha de publicación, likes y tags. """ videosfile = cf.data_dir + 'videos-large.csv' input_file = csv.DictReader(open(videosfile, encoding='utf-8')) for video in input_file: cada_video = {} datos_str = [ "title", "video_id", "channel_title", "country", "publish_time" ] datos_numeros = ["likes", "dislikes", "category_id", "views"] for dato in datos_str: cada_video[dato] = video[dato] for dato_num in datos_numeros: cada_video[dato_num] = video[dato_num] cada_video["trending_date"] = datetime.strptime( video['trending_date'], '%y.%d.%m').date() model.addVideo(catalog, cada_video) model.addCountry(catalog, cada_video) model.addCategory_id(catalog, cada_video)
def loadCountries(catalog): """ Carga los países del archivo. Por cada país se toma los datos necesarios: nombre del país, nombre de la capital, latitud de la capital y longitud de la capital """ videosfile = cf.data_dir + 'countries.csv' input_file = csv.DictReader(open(videosfile, encoding='utf-8')) rows = list(input_file) size = len(rows) for i, country in enumerate(rows): cada_country = { "country": country["CountryName"], "capital": country["CapitalName"], "latitude": float(country["CapitalLatitude"]), "longitude": float(country["CapitalLongitude"]), "population": country["Population"], "users": country["Internet users"] } if i == size - 1: ultimo_country = cada_country model.addCountry(catalog, cada_country) else: model.addCountry(catalog, cada_country) return ultimo_country
def loadData(analyzer, tupleFiles): connectionsFile = cf.data_dir + tupleFiles[0] connections_file = csv.DictReader(open(connectionsFile, encoding="utf-8-sig"), delimiter=",") countriesFile = cf.data_dir + tupleFiles[1] countries_file = csv.DictReader(open(countriesFile, encoding="utf-8"), delimiter=",") landing_pointsFile = cf.data_dir + tupleFiles[2] landing_point_file = csv.DictReader(open(landing_pointsFile, encoding="utf-8"), delimiter=",") for station in connections_file: model.addConnection(analyzer, station) model.createVertexEdge(analyzer, station) model.addConnectionToMap(analyzer) for country in countries_file: model.addCountry(analyzer, country) model.addCountryToMap(analyzer) for landingPoint in landing_point_file: model.addLandingPoint(analyzer, landingPoint) model.addLandingPointToMap(analyzer) model.addLandingPointToIDMap(analyzer) model.connectToCapital(analyzer) return analyzer
def newData(catalog): countriesFile = cf.data_dir + 'countries.csv' countries_input_file = csv.DictReader(open(countriesFile, encoding='utf-8-sig'), delimiter=',') landingPoints = cf.data_dir + 'landing_points.csv' ldPoints_input_file = csv.DictReader(open(landingPoints, encoding='utf-8-sig'), delimiter=',') connectionsFile = cf.data_dir + 'connections.csv' connectionsFile_input_file = csv.DictReader(open(connectionsFile, encoding='utf-8-sig'), delimiter=',') for line in countries_input_file: c = { 'CapitalName': line['CapitalName'], 'latitude': line['CapitalLatitude'], 'longitude': line['CapitalLongitude'] } md.addCountry(catalog, line['CountryName'], c) md.addReq(catalog, line['CountryName']) for line in ldPoints_input_file: ld = { 'name': line['name'].split(', '), 'latitude': line['latitude'], 'longitude': line['longitude'] } md.addLandingpoint(catalog, line['landing_point_id'], ld) md.addNewCity(catalog, line['landing_point_id'], ld['name']) for line in connectionsFile_input_file: md.addlpConnections(catalog, line) md.addCable(catalog, line) md.conf(catalog) md.sameLP(catalog)
def loadCountries(analyzer, countries_file): countries_file = cf.data_dir + countries_file input_file = csv.DictReader(open(countries_file, encoding='utf-8-sig'), delimiter=',') for country in input_file: model.addCountry(analyzer, country) model.addCountryMap(analyzer,country) return analyzer
def addElementsmapsDetails(catalog, file1, file2): """ Carga los elementos de los mapas relacionados con los details. """ file1 = config.file_dir + file1 dialect = csv.excel() dialect.delimiter = ";" input_file1 = csv.DictReader(open(file1, encoding="utf-8"), dialect=dialect) file2 = config.file_dir + file2 dialect = csv.excel() dialect.delimiter = ";" input_file2 = csv.DictReader(open(file2, encoding="utf-8"), dialect=dialect) for movie1 in input_file1: lt.addLast(catalog['Data']['details'], movie1) md.addCompany(movie1, catalog) md.addGenre(movie1, catalog) iterator = it.newIterator(catalog['Data']['details']) for movie2 in input_file2: movie1 = it.next(iterator) lt.addLast(catalog['Data']['casting'], movie2) md.addDirector(movie2, movie1, catalog) md.addActor(movie2, movie1, catalog) md.addCountry(movie2, movie1, catalog)
def loadCountry(analyzer, file): file = cf.data_dir + file input_file = csv.DictReader(open(file, encoding="utf-8"), delimiter=",") for country in input_file: model.addCountry(analyzer, country) return analyzer
def loadCountries(catalog): countriesfile = cf.data_dir + "countries.csv" inputfile = csv.DictReader(open(countriesfile, encoding='utf-8'), delimiter=",") lastcountry = None for country in inputfile: model.addCountry(catalog, country) lastcountry = country return lastcountry
def loadCountries(analyzer): """ Carga los paises """ file = cf.data_dir + 'countries.csv' input_file = csv.DictReader(open(file, encoding='utf-8'), delimiter=",") for country in input_file: model.addCountry(analyzer, country)
def loadCountries(analyzer, servicesfile): """ Carga los datos del archivo Countries CSV en el modelo. """ servicesfile = cf.data_dir + servicesfile input_file = csv.DictReader(open(servicesfile, encoding="utf-8-sig"), delimiter=",") for country in input_file: model.addCountry(analyzer, country) return analyzer
def loadVideos(catalog): """ Carga los videos del archivo al catalogo """ booksfile = cf.data_dir + 'videos-20pct.csv' input_file = csv.DictReader(open(booksfile, encoding='utf-8')) for video in input_file: model.addVideoToCat(catalog, video) model.addVideo(catalog, video) model.addCountry(catalog, video)
def loadCountries(analyzer, servicesfile): """ Carga los libros del archivo. Por cada libro se indica al modelo que debe adicionarlo al catalogo. """ videosfile = cf.data_dir + servicesfile input_file = csv.DictReader(open(videosfile, encoding='utf-8')) for country in input_file: model.addCountry(analyzer, country) return analyzer
def loadVideos(catalog): file_name = cf.data_dir + "videos-large.csv" input_file = csv.DictReader(open(file_name, encoding="utf-8")) i = 0 for video in input_file: i += 1 model.addVideo(catalog, video) model.addCategory_videos(catalog, video) model.addCountry(catalog, video) print("Cantidad de videos cargados: ", i)
def loadCountrys(analyzer): """ Carga los paises del archivo. """ etiquetasfile = cf.data_dir + 'countries.csv' input_file = csv.DictReader(open(etiquetasfile, encoding='utf-8')) ultimo = None for country in input_file: model.addCountry(analyzer, country) ultimo = country return ultimo
def loadServices(analyzer): """ Carga los datos de los archivos CSV en el modelo. Se crea un arco entre cada par de estaciones que pertenecen al mismo servicio y van en el mismo sentido. addRouteConnection crea conexiones entre diferentes rutas servidas en una misma estación. """ """ delta_time = -1.0 delta_memory = -1.0 tracemalloc.start() start_time = getTime() start_memory = getMemory() """ servicefile = 'connections.csv' servicesfile = cf.data_dir + servicefile input_file = csv.DictReader(open(servicesfile, encoding="utf-8"), delimiter=",") for line in input_file: model.addLanding(analyzer, line) servicefile = 'landing_points.csv' servicesfile = cf.data_dir + servicefile input_file = csv.DictReader(open(servicesfile, encoding="utf-8"), delimiter=",") lastservice = None for line in input_file: model.addInfo(analyzer, line) servicefile = 'countries.csv' servicesfile = cf.data_dir + servicefile input_file = csv.DictReader(open(servicesfile, encoding="utf-8"), delimiter=",") lastservice = None for line in input_file: model.addCountry(analyzer, line) print("1") model.addConnection(analyzer) print('2') model.addCapital(analyzer) print('3') """ stop_memory = getMemory() stop_time = getTime() tracemalloc.stop() delta_time = stop_time - start_time delta_memory = deltaMemory(start_memory, stop_memory) return (analyzer,delta_time, delta_memory)""" return analyzer
def loadCountries(cont, countriesfile): cfile = cf.data_dir + countriesfile input_file = csv.DictReader(open(cfile, encoding="utf-8"), delimiter=",") a = 1 b = {} for i in input_file: model.addCountry(cont, i) a += 1 if a == 239: b = i return b
def loadData(analyzer): delta_time = -1.0 delta_memory = -1.0 tracemalloc.start() start_time = getTime() start_memory = getMemory() songfile1 = cf.data_dir + 'PrepData/connections.csv' input_file1 = csv.DictReader(open(songfile1, encoding='utf-8-sig')) songfile2 = cf.data_dir + 'PrepData/countries.csv' input_file2 = csv.DictReader(open(songfile2, encoding='utf-8')) songfile3 = cf.data_dir + 'PrepData/landing_points.csv' input_file3 = csv.DictReader(open(songfile3, encoding='utf-8')) i = 1 for value in input_file2: model.addCountry(analyzer, value) if i == 236: country = model.selectResult(value, 'country') i += 1 i = 1 for value in input_file3: model.addLandingPoint(analyzer, value) if i == 1: landingpoint = model.selectResult(value, 'landingpoint') i += 1 for value in input_file1: model.addPointConnection(analyzer, value) model.addLandingPointConnections(analyzer) model.addCountryConnections(analyzer) stop_memory = getMemory() stop_time = getTime() tracemalloc.stop() delta_time = stop_time - start_time delta_memory = deltaMemory(start_memory, stop_memory) return ((country, landingpoint), (delta_time, delta_memory))
def loadCountries(analyzer): ''' Se carga la información del CSV de countries.csv ''' countriesFile = cf.data_dir + 'countries.csv' input_file = csv.DictReader(open(countriesFile, encoding='utf-8-sig'), delimiter=',') for country in input_file: # CountryName,CapitalName,CapitalLatitude,CapitalLongitude,CountryCode,ContinentName,Population,Internet users filtered_dict = { 'CountryName': country['CountryName'], 'CapitalName': country['CapitalName'], 'CapitalLatitude': (country['CapitalLatitude']), 'CapitalLongitude': (country['CapitalLongitude']), 'CountryCode': country['CountryCode'], 'ContinentName': country['ContinentName'], 'Population': float(country['Population'].replace('.', '')), 'Internet users': float(country['Internet users'].replace('.', '')) } if filtered_dict['CapitalLatitude'] is not '': filtered_dict['CapitalLatitude'] = float( filtered_dict['CapitalLatitude']) if filtered_dict['CapitalLongitude'] is not '': filtered_dict['CapitalLongitude'] = float( filtered_dict['CapitalLongitude']) if filtered_dict['CountryName'] is not '': model.addCountry(analyzer, filtered_dict) model.addCapitalLandingPoint(analyzer, filtered_dict) resultado = model.lastCountry(analyzer, analyzer['orderedCountries']) analyzer[ 'orderedCountries'] = None # Somos fancy y nos gusta el espacio en memoria return resultado
def cargar(analyzer): landingpointsfile = cf.data_dir + "landing_points.csv" input_file = csv.DictReader(open(landingpointsfile, encoding="utf-8"), delimiter=",") for point in input_file: model.addLandingPoint(analyzer,point) connectionsfile = cf.data_dir + "connections.csv" input_file = csv.DictReader(open(connectionsfile, encoding="utf-8"),delimiter=",") for connection in input_file: model.addLP_cable(analyzer, connection) countriesfile = cf.data_dir + "countries.csv" input_file = csv.DictReader(open(countriesfile, encoding="utf-8"), delimiter=",") for country in input_file: model.addCountry(analyzer,country) countriesfile2 = cf.data_dir + "countries.csv" input_file = csv.DictReader(open(countriesfile2, encoding="utf-8"), delimiter=",") for country2 in input_file: model.addCapital_V_E(analyzer,country2) model.edges_same_lp(analyzer) num_lps = mp.size(analyzer["landing_points"]) num_conexiones = gr.numEdges(analyzer["connections_distance"]) num_paises = mp.size(analyzer["countries"]) return num_lps,num_conexiones,num_paises
def loadInfo(structure): countries_count = 0 last_country = None countriesFile = cf.data_dir + 'countries.csv' countriesData = csv.DictReader(open(countriesFile, encoding="utf-8"), delimiter=",") for country in countriesData: structure = model.addCountry(country, structure) countries_count += 1 if countries_count == 238: last_country = country countries_info = {'last': last_country, 'count': countries_count} lpoints_count = 0 first_lpoint = None lpointsFile = cf.data_dir + 'landing_points.csv' lpointsData = csv.DictReader(open(lpointsFile, encoding="utf-8"), delimiter=",") for lpoint in lpointsData: structure = model.addLpoint(lpoint, structure) lpoints_count += 1 if lpoints_count == 1: first_lpoint = lpoint lpoints_info = {'first': first_lpoint, 'count': lpoints_count} connections_count = 0 connectionsFile = cf.data_dir + 'connections.csv' connectionsData = csv.DictReader(open(connectionsFile, encoding="utf-8-sig"), delimiter=",") for connection in connectionsData: structure = model.addConnection(connection, structure) connections_count += 1 connections_info = connections_count return [structure, [countries_info, lpoints_info, connections_info]]
def loadCountries(catalog, countriesfile): countriesfile = cf.data_dir + countriesfile input_file = csv.DictReader(open(countriesfile, encoding="utf-8-sig"), delimiter=",") for country in input_file: model.addCountry(catalog, country)
def loadVideos(catalog): videosfile = cf.data_dir + "videos-small.csv" input_file = csv.DictReader(open(videosfile, encoding='utf-8')) for video in input_file: model.addVideo(catalog, video) model.addCountry(catalog, video)
def loadCountries(catalog): contentfile = cf.data_dir + 'countries.csv' input_file = csv.DictReader(open(contentfile, encoding='utf-8')) for country in input_file: model.addCountry(catalog, country)
def loadCountries(catalog): filename = cf.data_dir + "countries.csv" input_file = csv.DictReader(open(filename, encoding="utf-8"), delimiter=",") for country in input_file: model.addCountry(catalog, country)
def loadData(analyzer, landingFile, connectionsFile, countriesFile): """ Carga los datos de los archivos CSV en el modelo. Se crea un arco entre cada par de estaciones que pertenecen al mismo servicio y van en el mismo sentido. addRouteConnection crea conexiones entre diferentes rutas servidas en una misma estación. """ landingFile = cf.data_dir + landingFile connectionsFile = cf.data_dir + connectionsFile countriesFile = cf.data_dir + countriesFile landing_File = csv.DictReader(open(landingFile, encoding="utf-8"), delimiter=",") connections_File = csv.DictReader(open(connectionsFile, encoding="utf-8-sig"), delimiter=",") countries_File = csv.DictReader(open(countriesFile, encoding="utf-8"), delimiter=",") # Creacion de listas vacias para agregar cada uno # de los elementos de los archivos # y asi recorrerlos mas facilmente. listaLandingPoints = lt.newList('ARRAY_LIST') listaConnections = lt.newList('ARRAY_LIST') listaCountries = lt.newList('ARRAY_LIST') for punto in landing_File: model.addPosition(analyzer, punto) model.addLandingPointInfo(analyzer, punto) lt.addLast(listaLandingPoints, punto) for connection in connections_File: lt.addLast(listaConnections, connection) for pais in countries_File: lt.addLast(listaCountries, pais) model.addCountry(analyzer, pais) for point in lt.iterator(listaLandingPoints): nombre = point['name'].split(", ") if len(nombre) < 2: pais = 'Micronesia' else: pais = nombre[1] for country in lt.iterator(listaCountries): if country['CountryName'] == pais: capital = country['CapitalName'] + "-" + pais model.addPoint(analyzer, capital) datosCapital = country break lstPuntoActual = None for connection in lt.iterator(listaConnections): if connection['origin'] == point['landing_point_id']: lstPuntoActual = model.addLandingPoint(analyzer, point, connection, lstPuntoActual, datosCapital) model.addConnectionsPoint(analyzer, lstPuntoActual) for country in lt.iterator(listaCountries): model.addCapitals(analyzer, country)
def loadVideos(catalog): videosfile = cf.data_dir + 'Videos/videos-large.csv' input_file = csv.DictReader(open(videosfile, encoding='utf-8')) for video in input_file: model.addCountry(catalog, video['country'], video) model.addVideo(catalog, video)