def loadVideos(catalog):
"""
Carga los videos del archivo. Por cada video se toman los datos necesarios:
video id, trending date, category id, views, nombre del canal, país, nombre del
video, likes, dislikes, fecha de publicación, likes y tags.
"""
videosfile = cf.data_dir + 'videos-large.csv'
input_file = csv.DictReader(open(videosfile, encoding='utf-8'))
for video in input_file:
cada_video = {}
datos_str = [
"title", "video_id", "channel_title", "country", "publish_time"
]
datos_numeros = ["likes", "dislikes", "category_id", "views"]
for dato in datos_str:
cada_video[dato] = video[dato]
for dato_num in datos_numeros:
cada_video[dato_num] = video[dato_num]
cada_video["trending_date"] = datetime.strptime(
video['trending_date'], '%y.%d.%m').date()
model.addVideo(catalog, cada_video)
model.addCountry(catalog, cada_video)
model.addCategory_id(catalog, cada_video)
def loadCountries(catalog):
"""
Carga los países del archivo. Por cada país se toma los datos necesarios:
nombre del país, nombre de la capital, latitud de la capital y longitud de la capital
"""
videosfile = cf.data_dir + 'countries.csv'
input_file = csv.DictReader(open(videosfile, encoding='utf-8'))
rows = list(input_file)
size = len(rows)
for i, country in enumerate(rows):
cada_country = {
"country": country["CountryName"],
"capital": country["CapitalName"],
"latitude": float(country["CapitalLatitude"]),
"longitude": float(country["CapitalLongitude"]),
"population": country["Population"],
"users": country["Internet users"]
}
if i == size - 1:
ultimo_country = cada_country
model.addCountry(catalog, cada_country)
else:
model.addCountry(catalog, cada_country)
return ultimo_country
def loadData(analyzer, tupleFiles):
connectionsFile = cf.data_dir + tupleFiles[0]
connections_file = csv.DictReader(open(connectionsFile,
encoding="utf-8-sig"),
delimiter=",")
countriesFile = cf.data_dir + tupleFiles[1]
countries_file = csv.DictReader(open(countriesFile, encoding="utf-8"),
delimiter=",")
landing_pointsFile = cf.data_dir + tupleFiles[2]
landing_point_file = csv.DictReader(open(landing_pointsFile,
encoding="utf-8"),
delimiter=",")
for station in connections_file:
model.addConnection(analyzer, station)
model.createVertexEdge(analyzer, station)
model.addConnectionToMap(analyzer)
for country in countries_file:
model.addCountry(analyzer, country)
model.addCountryToMap(analyzer)
for landingPoint in landing_point_file:
model.addLandingPoint(analyzer, landingPoint)
model.addLandingPointToMap(analyzer)
model.addLandingPointToIDMap(analyzer)
model.connectToCapital(analyzer)
return analyzer
def newData(catalog):
countriesFile = cf.data_dir + 'countries.csv'
countries_input_file = csv.DictReader(open(countriesFile,
encoding='utf-8-sig'),
delimiter=',')
landingPoints = cf.data_dir + 'landing_points.csv'
ldPoints_input_file = csv.DictReader(open(landingPoints,
encoding='utf-8-sig'),
delimiter=',')
connectionsFile = cf.data_dir + 'connections.csv'
connectionsFile_input_file = csv.DictReader(open(connectionsFile,
encoding='utf-8-sig'),
delimiter=',')
for line in countries_input_file:
c = {
'CapitalName': line['CapitalName'],
'latitude': line['CapitalLatitude'],
'longitude': line['CapitalLongitude']
}
md.addCountry(catalog, line['CountryName'], c)
md.addReq(catalog, line['CountryName'])
for line in ldPoints_input_file:
ld = {
'name': line['name'].split(', '),
'latitude': line['latitude'],
'longitude': line['longitude']
}
md.addLandingpoint(catalog, line['landing_point_id'], ld)
md.addNewCity(catalog, line['landing_point_id'], ld['name'])
for line in connectionsFile_input_file:
md.addlpConnections(catalog, line)
md.addCable(catalog, line)
md.conf(catalog)
md.sameLP(catalog)
def loadCountries(analyzer, countries_file):
countries_file = cf.data_dir + countries_file
input_file = csv.DictReader(open(countries_file, encoding='utf-8-sig'), delimiter=',')
for country in input_file:
model.addCountry(analyzer, country)
model.addCountryMap(analyzer,country)
return analyzer
def addElementsmapsDetails(catalog, file1, file2):
"""
Carga los elementos de los mapas relacionados con los details.
"""
file1 = config.file_dir + file1
dialect = csv.excel()
dialect.delimiter = ";"
input_file1 = csv.DictReader(open(file1, encoding="utf-8"),
dialect=dialect)
file2 = config.file_dir + file2
dialect = csv.excel()
dialect.delimiter = ";"
input_file2 = csv.DictReader(open(file2, encoding="utf-8"),
dialect=dialect)
for movie1 in input_file1:
lt.addLast(catalog['Data']['details'], movie1)
md.addCompany(movie1, catalog)
md.addGenre(movie1, catalog)
iterator = it.newIterator(catalog['Data']['details'])
for movie2 in input_file2:
movie1 = it.next(iterator)
lt.addLast(catalog['Data']['casting'], movie2)
md.addDirector(movie2, movie1, catalog)
md.addActor(movie2, movie1, catalog)
md.addCountry(movie2, movie1, catalog)
def loadCountry(analyzer, file):
file = cf.data_dir + file
input_file = csv.DictReader(open(file, encoding="utf-8"), delimiter=",")
for country in input_file:
model.addCountry(analyzer, country)
return analyzer
def loadCountries(catalog):
countriesfile = cf.data_dir + "countries.csv"
inputfile = csv.DictReader(open(countriesfile, encoding='utf-8'), delimiter=",")
lastcountry = None
for country in inputfile:
model.addCountry(catalog, country)
lastcountry = country
return lastcountry
def loadCountries(analyzer):
"""
Carga los paises
"""
file = cf.data_dir + 'countries.csv'
input_file = csv.DictReader(open(file, encoding='utf-8'), delimiter=",")
for country in input_file:
model.addCountry(analyzer, country)
def loadCountries(analyzer, servicesfile):
"""
Carga los datos del archivo Countries CSV en el modelo.
"""
servicesfile = cf.data_dir + servicesfile
input_file = csv.DictReader(open(servicesfile, encoding="utf-8-sig"),
delimiter=",")
for country in input_file:
model.addCountry(analyzer, country)
return analyzer
def loadVideos(catalog):
"""
Carga los videos del archivo al catalogo
"""
booksfile = cf.data_dir + 'videos-20pct.csv'
input_file = csv.DictReader(open(booksfile, encoding='utf-8'))
for video in input_file:
model.addVideoToCat(catalog, video)
model.addVideo(catalog, video)
model.addCountry(catalog, video)
def loadCountries(analyzer, servicesfile):
"""
Carga los libros del archivo. Por cada libro se indica al
modelo que debe adicionarlo al catalogo.
"""
videosfile = cf.data_dir + servicesfile
input_file = csv.DictReader(open(videosfile, encoding='utf-8'))
for country in input_file:
model.addCountry(analyzer, country)
return analyzer
def loadVideos(catalog):
file_name = cf.data_dir + "videos-large.csv"
input_file = csv.DictReader(open(file_name, encoding="utf-8"))
i = 0
for video in input_file:
i += 1
model.addVideo(catalog, video)
model.addCategory_videos(catalog, video)
model.addCountry(catalog, video)
print("Cantidad de videos cargados: ", i)
def loadCountrys(analyzer):
"""
Carga los paises del archivo.
"""
etiquetasfile = cf.data_dir + 'countries.csv'
input_file = csv.DictReader(open(etiquetasfile, encoding='utf-8'))
ultimo = None
for country in input_file:
model.addCountry(analyzer, country)
ultimo = country
return ultimo
def loadServices(analyzer):
"""
Carga los datos de los archivos CSV en el modelo.
Se crea un arco entre cada par de estaciones que
pertenecen al mismo servicio y van en el mismo sentido.
addRouteConnection crea conexiones entre diferentes rutas
servidas en una misma estación.
"""
"""
delta_time = -1.0
delta_memory = -1.0
tracemalloc.start()
start_time = getTime()
start_memory = getMemory()
"""
servicefile = 'connections.csv'
servicesfile = cf.data_dir + servicefile
input_file = csv.DictReader(open(servicesfile, encoding="utf-8"),
delimiter=",")
for line in input_file:
model.addLanding(analyzer, line)
servicefile = 'landing_points.csv'
servicesfile = cf.data_dir + servicefile
input_file = csv.DictReader(open(servicesfile, encoding="utf-8"),
delimiter=",")
lastservice = None
for line in input_file:
model.addInfo(analyzer, line)
servicefile = 'countries.csv'
servicesfile = cf.data_dir + servicefile
input_file = csv.DictReader(open(servicesfile, encoding="utf-8"),
delimiter=",")
lastservice = None
for line in input_file:
model.addCountry(analyzer, line)
print("1")
model.addConnection(analyzer)
print('2')
model.addCapital(analyzer)
print('3')
"""
stop_memory = getMemory()
stop_time = getTime()
tracemalloc.stop()
delta_time = stop_time - start_time
delta_memory = deltaMemory(start_memory, stop_memory)
return (analyzer,delta_time, delta_memory)"""
return analyzer
def loadCountries(cont, countriesfile):
cfile = cf.data_dir + countriesfile
input_file = csv.DictReader(open(cfile, encoding="utf-8"), delimiter=",")
a = 1
b = {}
for i in input_file:
model.addCountry(cont, i)
a += 1
if a == 239:
b = i
return b
def loadData(analyzer):
delta_time = -1.0
delta_memory = -1.0
tracemalloc.start()
start_time = getTime()
start_memory = getMemory()
songfile1 = cf.data_dir + 'PrepData/connections.csv'
input_file1 = csv.DictReader(open(songfile1, encoding='utf-8-sig'))
songfile2 = cf.data_dir + 'PrepData/countries.csv'
input_file2 = csv.DictReader(open(songfile2, encoding='utf-8'))
songfile3 = cf.data_dir + 'PrepData/landing_points.csv'
input_file3 = csv.DictReader(open(songfile3, encoding='utf-8'))
i = 1
for value in input_file2:
model.addCountry(analyzer, value)
if i == 236:
country = model.selectResult(value, 'country')
i += 1
i = 1
for value in input_file3:
model.addLandingPoint(analyzer, value)
if i == 1:
landingpoint = model.selectResult(value, 'landingpoint')
i += 1
for value in input_file1:
model.addPointConnection(analyzer, value)
model.addLandingPointConnections(analyzer)
model.addCountryConnections(analyzer)
stop_memory = getMemory()
stop_time = getTime()
tracemalloc.stop()
delta_time = stop_time - start_time
delta_memory = deltaMemory(start_memory, stop_memory)
return ((country, landingpoint), (delta_time, delta_memory))
def loadCountries(analyzer):
'''
Se carga la información del CSV de countries.csv
'''
countriesFile = cf.data_dir + 'countries.csv'
input_file = csv.DictReader(open(countriesFile, encoding='utf-8-sig'),
delimiter=',')
for country in input_file:
# CountryName,CapitalName,CapitalLatitude,CapitalLongitude,CountryCode,ContinentName,Population,Internet users
filtered_dict = {
'CountryName': country['CountryName'],
'CapitalName': country['CapitalName'],
'CapitalLatitude': (country['CapitalLatitude']),
'CapitalLongitude': (country['CapitalLongitude']),
'CountryCode': country['CountryCode'],
'ContinentName': country['ContinentName'],
'Population': float(country['Population'].replace('.', '')),
'Internet users': float(country['Internet users'].replace('.', ''))
}
if filtered_dict['CapitalLatitude'] is not '':
filtered_dict['CapitalLatitude'] = float(
filtered_dict['CapitalLatitude'])
if filtered_dict['CapitalLongitude'] is not '':
filtered_dict['CapitalLongitude'] = float(
filtered_dict['CapitalLongitude'])
if filtered_dict['CountryName'] is not '':
model.addCountry(analyzer, filtered_dict)
model.addCapitalLandingPoint(analyzer, filtered_dict)
resultado = model.lastCountry(analyzer, analyzer['orderedCountries'])
analyzer[
'orderedCountries'] = None # Somos fancy y nos gusta el espacio en memoria
return resultado
def cargar(analyzer):
landingpointsfile = cf.data_dir + "landing_points.csv"
input_file = csv.DictReader(open(landingpointsfile, encoding="utf-8"),
delimiter=",")
for point in input_file:
model.addLandingPoint(analyzer,point)
connectionsfile = cf.data_dir + "connections.csv"
input_file = csv.DictReader(open(connectionsfile, encoding="utf-8"),delimiter=",")
for connection in input_file:
model.addLP_cable(analyzer, connection)
countriesfile = cf.data_dir + "countries.csv"
input_file = csv.DictReader(open(countriesfile, encoding="utf-8"),
delimiter=",")
for country in input_file:
model.addCountry(analyzer,country)
countriesfile2 = cf.data_dir + "countries.csv"
input_file = csv.DictReader(open(countriesfile2, encoding="utf-8"),
delimiter=",")
for country2 in input_file:
model.addCapital_V_E(analyzer,country2)
model.edges_same_lp(analyzer)
num_lps = mp.size(analyzer["landing_points"])
num_conexiones = gr.numEdges(analyzer["connections_distance"])
num_paises = mp.size(analyzer["countries"])
return num_lps,num_conexiones,num_paises
def loadInfo(structure):
countries_count = 0
last_country = None
countriesFile = cf.data_dir + 'countries.csv'
countriesData = csv.DictReader(open(countriesFile, encoding="utf-8"),
delimiter=",")
for country in countriesData:
structure = model.addCountry(country, structure)
countries_count += 1
if countries_count == 238:
last_country = country
countries_info = {'last': last_country, 'count': countries_count}
lpoints_count = 0
first_lpoint = None
lpointsFile = cf.data_dir + 'landing_points.csv'
lpointsData = csv.DictReader(open(lpointsFile, encoding="utf-8"),
delimiter=",")
for lpoint in lpointsData:
structure = model.addLpoint(lpoint, structure)
lpoints_count += 1
if lpoints_count == 1:
first_lpoint = lpoint
lpoints_info = {'first': first_lpoint, 'count': lpoints_count}
connections_count = 0
connectionsFile = cf.data_dir + 'connections.csv'
connectionsData = csv.DictReader(open(connectionsFile,
encoding="utf-8-sig"),
delimiter=",")
for connection in connectionsData:
structure = model.addConnection(connection, structure)
connections_count += 1
connections_info = connections_count
return [structure, [countries_info, lpoints_info, connections_info]]
def loadCountries(catalog, countriesfile):
countriesfile = cf.data_dir + countriesfile
input_file = csv.DictReader(open(countriesfile, encoding="utf-8-sig"),
delimiter=",")
for country in input_file:
model.addCountry(catalog, country)
def loadVideos(catalog):
videosfile = cf.data_dir + "videos-small.csv"
input_file = csv.DictReader(open(videosfile, encoding='utf-8'))
for video in input_file:
model.addVideo(catalog, video)
model.addCountry(catalog, video)
def loadCountries(catalog):
contentfile = cf.data_dir + 'countries.csv'
input_file = csv.DictReader(open(contentfile, encoding='utf-8'))
for country in input_file:
model.addCountry(catalog, country)
def loadCountries(catalog):
filename = cf.data_dir + "countries.csv"
input_file = csv.DictReader(open(filename, encoding="utf-8"),
delimiter=",")
for country in input_file:
model.addCountry(catalog, country)
def loadData(analyzer, landingFile, connectionsFile, countriesFile):
"""
Carga los datos de los archivos CSV en el modelo.
Se crea un arco entre cada par de estaciones que
pertenecen al mismo servicio y van en el mismo sentido.
addRouteConnection crea conexiones entre diferentes rutas
servidas en una misma estación.
"""
landingFile = cf.data_dir + landingFile
connectionsFile = cf.data_dir + connectionsFile
countriesFile = cf.data_dir + countriesFile
landing_File = csv.DictReader(open(landingFile, encoding="utf-8"),
delimiter=",")
connections_File = csv.DictReader(open(connectionsFile,
encoding="utf-8-sig"),
delimiter=",")
countries_File = csv.DictReader(open(countriesFile, encoding="utf-8"),
delimiter=",")
# Creacion de listas vacias para agregar cada uno
# de los elementos de los archivos
# y asi recorrerlos mas facilmente.
listaLandingPoints = lt.newList('ARRAY_LIST')
listaConnections = lt.newList('ARRAY_LIST')
listaCountries = lt.newList('ARRAY_LIST')
for punto in landing_File:
model.addPosition(analyzer, punto)
model.addLandingPointInfo(analyzer, punto)
lt.addLast(listaLandingPoints, punto)
for connection in connections_File:
lt.addLast(listaConnections, connection)
for pais in countries_File:
lt.addLast(listaCountries, pais)
model.addCountry(analyzer, pais)
for point in lt.iterator(listaLandingPoints):
nombre = point['name'].split(", ")
if len(nombre) < 2:
pais = 'Micronesia'
else:
pais = nombre[1]
for country in lt.iterator(listaCountries):
if country['CountryName'] == pais:
capital = country['CapitalName'] + "-" + pais
model.addPoint(analyzer, capital)
datosCapital = country
break
lstPuntoActual = None
for connection in lt.iterator(listaConnections):
if connection['origin'] == point['landing_point_id']:
lstPuntoActual = model.addLandingPoint(analyzer, point,
connection,
lstPuntoActual,
datosCapital)
model.addConnectionsPoint(analyzer, lstPuntoActual)
for country in lt.iterator(listaCountries):
model.addCapitals(analyzer, country)
def loadVideos(catalog):
videosfile = cf.data_dir + 'Videos/videos-large.csv'
input_file = csv.DictReader(open(videosfile, encoding='utf-8'))
for video in input_file:
model.addCountry(catalog, video['country'], video)
model.addVideo(catalog, video)
