def display_pick(pick):
# Note: this does noting really, commit out if you dont want wait for the loading bar
spinner = PixelSpinner('Finding a movie! ')
for i in range(10):
time.sleep(.3)
spinner.next()
print(f'\nGet ready to watch! {pick[1]}')
def save_response_content(response, destination):
CHUNK_SIZE = 32768
with open(destination, "wb") as f:
sp = Spinner("Downloading ... ")
for chunk in response.iter_content(CHUNK_SIZE):
if chunk:
f.write(chunk)
sp.next()
class Animate:
"""handles progress bars and spinners"""
def __init__(self, num):
self.bar = Bar(max=num)
self.spinner = PixelSpinner()
def next(self):
"""advance the spinner and bar to the next iteration in cycle"""
self.bar.next()
self.spinner.next()
def done(self):
"""complete the bar"""
self.bar.finish()
def mergeHashtags(self):
"""Merge the raw_hashtags table from the leaderboard into osmstats"""
log = PixelSpinner(
"Merging leaderboard hashtags table into Galaxy osmstats database. this may take a will..."
)
self.timer.start()
inquery = "SELECT changeset_id,hashtag FROM raw_changesets_hashtags INNER JOIN raw_hashtags ON (raw_changesets_hashtags.hashtag_id = id);"
self.indb.dbcursor.execute(inquery)
self.timer.stop()
for entry in self.indb.dbcursor.fetchall():
log.next()
outquery = "INSERT INTO changesets(id,hashtags) VALUES({id}, ARRAY['{hashtags}']) ON CONFLICT(id) DO UPDATE SET id="
# watch out for single quotes in user names
fixed = entry[1].replace("'", "'")
outquery += str(
entry[0]
) + ", hashtags=ARRAY_APPEND(changesets.hashtags, '" + fixed + "')"
self.outdb.dbcursor.execute(
outquery.format(id=int(entry[0]), hashtags=fixed))
def __init__(self, indb=None, outdb=None, host=None):
"""Load a small bounding box for each country using the modified raw_countries
table. Included in the source code for osm-stats-workers is a GeoJson file with
the boundries used to display country boundaries. As those boundaries were only
used by the front end, the boundaries are not in the database. The modified
raw_countries table is the same data with a new column added for the geometry."""
if indb is None:
indb = "leaderboard"
self.indb = Pydb(indb, host)
self.countries = dict()
geoquery = "SELECT id,St_AsText(ST_Envelope(ST_Buffer(ST_Centroid(boundary), 1, 4))) FROM raw_countries;"
log = PixelSpinner("Loading Country boundaries...")
self.indb.dbcursor.execute(geoquery)
for row in self.indb.dbcursor.fetchall():
log.next()
self.countries[row[0]] = row[1]
self.timer = Timer()
if outdb is None:
outdb = "galaxy"
self.outdb = Pydb(outdb, host)
connect = "dbname='" + options['database'] + "'"
dbshell = psycopg2.connect(connect)
dbshell.autocommit = True
dbcursor = dbshell.cursor()
osm = OsmFile("example.osm")
bar = PixelSpinner('Processing... ' + options["database"])
# Get all the highways
# query = """SELECT tags->'name',nodes,tags,ST_AsEWKT(linestring) FROM ways WHERE tags->'highway' is not NULL AND tags->'highway'!='path' LIMIT 5;"""
query = """SELECT id,tags->'name',nodes,tags FROM ways WHERE tags->'highway' is not NULL AND tags->'highway'!='path';"""
dbcursor.execute(query)
all = dbcursor.fetchall()
for line in all:
bar.next()
result = dict()
result['id'] = line[0]
result['name'] = line[1]
result['nodes'] = line[2]
result['tags'] = line[3]
# print("WAY: %s" % result['tags'])
# Get the data for each node in the array
if type(result['nodes']) != int:
for data in result['nodes']:
# print(data)
# if data:
query = """SELECT id,version,user_id,tstamp,changeset_id,tags,geom FROM nodes WHERE id=%d""" % data
# print(query)
dbcursor.execute(query)
def cli_login():
global session, globalError, semestres_final
art = """
_____ __ __ __ ___ ___ __ __ ________ __
||__ / ` | \|/ _`|| /\ | |__ \_/|__)/ \|__)||__ |__)
||___\__, |__/|\__>||/~~\|___ |___/ \| \__/| \||___| \ """
print(art)
print("Exportador de archivos del TEC Digital")
print("Creado por Joseph Vargas - https://twitter.com/JosephTico\n\n")
print("Ingrese sus credenciales del TEC Digital y presione Enter.")
if "TEC_USERNAME" in environ:
username = environ.get('TEC_USERNAME')
else:
username = input("Usuario: ").strip()
if "TEC_PASSWORD" in environ:
password = environ.get('TEC_PASSWORD')
else:
password = getpass.getpass("Contraseña: ")
spinner = PixelSpinner('Iniciando sesión... ')
thread = threading.Thread(target=td_login,args=(username,password))
thread.start()
while thread.is_alive() and globalError == False:
spinner.next()
thread.join()
if globalError:
return
print("\n")
print('Obteniendo cursos... ')
thread = threading.Thread(target=obtener_cursos)
thread.start()
thread.join()
if globalError:
return
print("\n")
print("Se han cargado satisfactoriamente los siguientes cursos:")
for semestre in semestres_final:
print("# " + semestre["titulo"])
for curso in semestre["cursos"]:
print("-- " + curso["titulo"])
print("\n")
if "AUTO_DOWNLOAD" not in environ and not query_yes_no("¿Desea iniciar la descarga de todos los archivos en la carpeta actual?"):
return
for semestre in semestres_final:
print("Descargando cursos de " + semestre["titulo"] + "...")
if not os.path.exists(semestre["titulo"]):
os.makedirs(semestre["titulo"])
for curso in semestre["cursos"]:
for attempt in range(5):
try:
print("Descargando archivos de " + curso["titulo"] + "...")
url = curso["url"] + "/download-archive?object_id=" + curso["folder_id"]
response = session.get(url, stream=True)
total_size_in_bytes= int(response.headers.get('content-length', 0))
block_size = 1024 #1 Kibibyte
progress_bar = tqdm(total=total_size_in_bytes, unit='iB', unit_scale=True)
filename = os.path.join(dirname, semestre["titulo"], curso["titulo"] + ".zip")
with open(filename, 'wb') as file:
for data in response.iter_content(block_size):
progress_bar.update(len(data))
file.write(data)
if total_size_in_bytes != 0 and progress_bar.n != total_size_in_bytes:
raise Exception('Error al descargar el archivo.')
try:
with zipfile.ZipFile(filename,"r") as zip_ref:
zip_ref.extractall(os.path.join(dirname, semestre["titulo"]))
except:
print("\nHa ocurrido un error al descomprimir los contenidos de este curso. Se mantendrá el archivo comprimido")
else:
os.remove(filename)
os.remove(filename)
progress_bar.close()
except KeyboardInterrupt:
sys.exit()
except:
print("\n\nERROR: Error al descargar el archivo. \nReintentando...")
else:
break
else:
print("Ha ocurrido un error al descargar el curso " + curso["titulo"] + ".\nSaltando...")
print("\n")
print("Proceso finalizado.")
def mergeStatistics(self, timestamp):
"""Merge the raw_changesets table from the leaderboard into osmstats"""
log = PixelSpinner(
"Merging leaderboard statistics into Galaxy osmstats database")
log.next()
self.timer.start()
query = "SELECT id, road_km_added, road_km_modified, waterway_km_added, waterway_km_modified, roads_added, roads_modified, waterways_added, waterways_modified, buildings_added, buildings_modified, pois_added, pois_modified, editor, user_id, created_at, closed_at, updated_at,country_id FROM raw_changesets INNER JOIN raw_changesets_countries ON (raw_changesets_countries.changeset_id = id);"
self.indb.dbcursor.execute(query)
self.timer.stop()
result = self.indb.dbcursor.fetchone()
while result is not None:
stats = dict()
added = dict()
modified = dict()
# non statistics fields
stats['change_id'] = result[0]
stats['editor'] = result[11]
stats['user_id'] = result[14]
stats['created_at'] = result[15]
stats['closed_at'] = result[16]
if stats['created_at'] is None and stats['closed_at'] is None:
result = self.indb.dbcursor.fetchone()
continue
if stats['created_at'] is None:
stats['created_at'] = stats['closed_at']
if stats['closed_at'] is None:
stats['closed_at'] = stats['created_at']
stats['updated_at'] = result[17]
stats['country_id'] = result[18]
if self.getBbox(result[18]) is None:
logging.warning(
"Country ID %s is not in the geoboundaries table" %
result[18])
result = self.indb.dbcursor.fetchone()
continue
stats['bbox'] = "ST_Multi(ST_GeomFromText('"
stats['bbox'] += self.getBbox(result[18]) + "')"
# Added fields
added['highway_km'] = result[1]
added['waterway_km'] = result[3]
added['highways'] = result[4]
added['waterways'] = result[7]
added['buildings'] = result[9]
added['pois'] = result[11]
# Modified fields
modified['highway_km'] = result[2]
modified['waterway_km'] = result[4]
modified['highways'] = result[6]
modified['waterways'] = result[8]
modified['buildings'] = result[10]
modified['pois'] = result[12]
# Get the next row, since we're done with this one
result = self.indb.dbcursor.fetchone()
# Build the hstore for the added statistics
hadd = "HSTORE(ARRAY["
for key, value in added.items():
hadd += "ARRAY['" + key + "','" + str(value) + "'],"
length = len(hadd) - 1
hadd = hadd[:length]
hadd += "])"
# Build the hstore for the added statistics
hmod = "HSTORE(ARRAY["
for key, value in modified.items():
hmod += "ARRAY['" + key + "','" + str(value) + "'],"
length = len(hmod) - 1
hmod = hmod[:length]
hmod += "])"
query = "INSERT INTO changesets(id, editor, user_id, created_at, closed_at, updated_at, added, modified, bbox)"
query += " VALUES({id}, '{editor}', {user_id}, '{created_at}', '{closed_at}', '{updated_at}', {add}, {mod}, {bbox})) ON CONFLICT(id) DO UPDATE SET editor='{editor}', user_id={user_id}, created_at='{created_at}', closed_at='{closed_at}', updated_at='{updated_at}', added={add}, modified={mod}, bbox={bbox});"
outquery = query.format(id=stats['change_id'],
editor=stats['editor'],
user_id=stats['user_id'],
created_at=stats['created_at'],
closed_at=stats['closed_at'],
updated_at=stats['updated_at'],
bbox=stats['bbox'],
add=hadd,
mod=hmod)
#print(outquery)
self.outdb.dbcursor.execute(outquery)
],
stderr=subprocess.STDOUT,
universal_newlines=True).strip()
bar.goto(current_time)
bar.finish()
progress_start = time.time()
video_files = []
video_files_populated = False
spinner = PixelSpinner('Please wait while generating video files list ')
while not video_files_populated:
for file in glob.iglob('**/*.*', recursive=True):
split_path_name = file.split("/")
folder = save_to + split_path_name[0]
if not os.path.isdir(file):
if magic.from_file(file, mime=True)[:5] == 'video':
video_files.append(file)
spinner.next()
video_files_populated = True
for video in range(len(video_files)):
generate_gif(video_files[video])
progress_end = time.time()
total_run = int(progress_end - progress_start)
print("== FINISHED ==")
print("Total Runtime: " + str(total_run) + " seconds")
