def from_parse_results(cls, result, assume_type=None):
""" Takes values from the pyparsing results and converts them to the appropriate internal objects """
# assumes that all three (hours, minutes, am_pm) are the same length
res_dct = result.asDict()
if "starttime" in res_dct and "endtime" in res_dct:
logger.info("time range detected")
start = res_dct.get("starttime")[0]
starttime = Time.from_parse_results(start)
end = res_dct.get("endtime")[0]
endtime = Time.from_parse_results(end)
if starttime.is_unknown() and assume_type is not None:
starttime.set_type(assume_type)
if endtime.is_unknown() and assume_type is not None:
endtime.set_type(assume_type)
if starttime.is_am() and endtime.is_am() and starttime.get_hours() > endtime.get_hours():
endtime.set_type(TimeType.PM)
return cls(starttime, endtime)
elif "time_shortcuts" in res_dct:
logger.info("time shortcut detected")
return cls.from_shortcut_string(result.get("time_shortcuts")[0])
else:
logger.info("unspecified time pattern detected")
logger.debug(vars(result))
# nothing specified, assumeit means every day
return cls(Time(0, 0, TimeType.AM), Time(11, 59, TimeType.PM))
def get(self):
# do not run on weekends and in the future holidays. app engine's cron does not allow to specify this
if Time.today_is_holiday():
return
interval = int(self.request.get('interval', self.default_interval))
timestamp = Time.current_time()
poller = Poller(interval)
calculator = Calculator()
# search all tweets and collect them in the calculator
poller.search(calculator)
# once we are done collecting all tweets, calculate the final score
calculator.calculate_score()
Score.insert(self.symbol, timestamp, calculator.score,
calculator.volume)
def select_all_times():
results = run_sql("SELECT * FROM times_available")
times = []
for result in results:
club = club_repository.select_club(result["club_id"])
time = Time(result["start_time"], result["duration"], club,
result["id"])
times.append(time)
return times
def select_time(id):
sql = "SELECT FROM times_available WHERE id=%s"
value = [id]
result = run_sql(sql, value)[0]
if result is not None:
club = club_repository.select_club(result["club_id"])
time = Time(result["start_time"], result["duration"], club,
result["id"])
return time
def from_shortcut_string(cls, times_shortcut, assume_type=None):
"""
create a times object from a shortcut string
"""
logger.debug("creating times object from shortcut: " + times_shortcut)
if times_shortcut is None:
raise TypeError("Cannot create Times Object from value None")
day = times_shortcut.lower()
# set up some shortcut ranges
allday = cls(Time(0, 0, TimeType.AM), Time(11, 59, TimeType.PM))
if "all day" in day:
return allday
elif "24" in day:
return allday
elif day == "":
# if no day is specified, assume the intention is all day
return allday
raise ValueError("string '" + times_shortcut + "' does not match a known pattern")
def from_shortcut_string(cls, times_string, assume_type=None):
"""
create a time object from a string
"""
logger.debug("creating times object from shortcut: " + times_string)
if times_string is None:
raise TypeError("Cannot create Times Object from value None")
day = times_string.lower()
# set up some shortcut ranges
allday = cls(Time(0, 0, TimeType.AM), Time(11, 59, TimeType.PM))
workhours = cls(Time(9, 0, TimeType.AM), Time(5, 0, TimeType.PM))
if "24" in day:
return allday
elif "business" in day:
return workhours
elif "work" in day:
return workhours
elif "all day" in day:
return allday
raise ValueError("string '" + times_string + "' does not match a known pattern")
def calculate_since(self):
since = Time.seconds_ago(self.interval)
return since
def calculate_since(self):
since = Time.seconds_ago(self.interval)
return since
def main():
## Se procesan los archivos 'data/log_data/'
#Se obtienen los nombre de todos los archivos los archivos
files = get_nameFiles('data/song_data/')
#Se crea un dataframe con la información de todos los archivos
df_song_data = files_To_DataFrame(files)
#Se obtienen los datos para la tabla SONG
df_song = df_song_data[[
'song_id', 'artist_id', 'title', 'year', 'duration'
]].dropna(subset=['song_id']).drop_duplicates(['song_id'])
#Se obtienen los datos para la tabla ARTIST
df_artist = df_song_data[[
'artist_id', 'artist_name', 'artist_location', 'artist_latitude',
'artist_longitude'
]].dropna(subset=['artist_id']).drop_duplicates(['artist_id'])
df_artist = df_artist.rename(
columns={
'artist_name': 'name',
'artist_location': 'location',
'artist_latitude': 'latitude',
'artist_longitude': 'longitude'
})
#Se procesan los archivos 'data/log_data/'
#Se obtienen los nombre de todos los archivos los archivos
files = get_nameFiles('data/log_data/')
#Se crea un dataframe con la información de todos los archivos
df_long_data = files_To_DataFrame(files)
#Se obtienen los datos para la tabla TIME_FORMAT
ts = pd.to_datetime(df_long_data['ts'], unit='ms')
timeFormat = (df_long_data['ts'].values, ts.dt.hour.values,
ts.dt.day.values, ts.dt.week.values, ts.dt.month.values,
ts.dt.year.values, ts.dt.weekday.values)
column_labels = [
'start_time', 'hour', 'day', 'week', 'month', 'year', 'weekday'
]
df_time_format = pd.DataFrame(data=list(zip(*timeFormat)),
columns=column_labels)
df_time_format = df_time_format.drop_duplicates(['start_time'])
#Se obtiene los datos para la tabla USER
convert_dict = {'user_id': int}
df_user = df_long_data[[
'userId', 'firstName', 'lastName', 'gender', 'level'
]].rename(columns={
'userId': 'user_id'
}).replace('', np.nan).dropna(subset=['user_id'])
df_user = df_user.astype(convert_dict)
df_user = df_user.drop_duplicates(['user_id'])
#Se obtiene los datos para la table SONG_PLAY
df_song_data_aux = df_song_data[[
'artist_id', 'song_id', 'title', 'artist_name'
]].rename(columns={
'title': 'song',
'artist_name': 'artist'
})
df_long_data_aux = df_long_data[[
'userId', 'sessionId', 'userAgent', 'level', 'location', 'ts',
'artist', 'song'
]]
df_song_play_join = pd.merge(df_long_data_aux,
df_song_data_aux,
on=['song', 'artist'],
how='left')
df_song_play_join = df_song_play_join.rename(
columns={
'ts': 'start_time',
'sessionId': 'session_id',
'userAgent': 'user_agent',
'userId': 'user_id'
}).replace('', 0)
#Se crea el indice songplay_id
df_song_play_join['songplay_id'] = df_song_play_join.index
#Se crean los objetos para ser guardados en BD
listObjectSong = [
Song(**kwargs) for kwargs in df_song.to_dict(orient='records')
]
listObjectArtist = [
Artist(**kwargs) for kwargs in df_artist.to_dict(orient='records')
]
listObjectTime = [
Time(**kwargs) for kwargs in df_time_format.to_dict(orient='records')
]
listObjectUser = [
User(**kwargs) for kwargs in df_user.to_dict(orient='records')
]
listObjectSongPlay = [
SongPlay(**kwargs)
for kwargs in df_song_play_join.to_dict(orient='records')
]
#Se eliminan las tablas, si existen y se crean
Base.metadata.drop_all(engine)
Base.metadata.create_all(engine)
#Se guardan los objetos
try:
session.bulk_save_objects(listObjectSong)
session.bulk_save_objects(listObjectArtist)
session.bulk_save_objects(listObjectTime)
session.bulk_save_objects(listObjectUser)
session.bulk_save_objects(listObjectSongPlay)
session.commit()
session.close()
except exc.SQLAlchemyError as ex:
session.close()
print('Exeption:', ex)
df_song_play_join = df_song_play_join.drop(['song', 'artist', 'length'],
axis=1)
def test_equals(self): allday = Times(Time(0, 0, TimeType.AM), Time(11, 59, TimeType.PM)) with self.assertRaises(NotImplementedError): allday == "something else" self.assertEqual(allday, allday)
**{
'id': i[0],
'name': i[1],
'img_folder_name': i[2],
'calories': i[3],
'instruction': i[4],
'time_id': i[5],
'history': i[6],
'advice': i[7]
})
session3.add(record)
data3 = Load_Data_3("csv/1time.csv")
for i in data3:
record = Time(**{'id': i[0], 'minutes': i[1]})
session3.add(record)
data4 = Load_Data_1("csv/1categories.csv")
for i in data4:
record = Categories(**{'id': i[0], 'category_name': i[1]})
session3.add(record)
data5 = Load_Data_1("csv/1cuisine.csv")
for i in data5:
record = Cuisine(**{'id': i[0], 'cuisine_name': i[1]})
session3.add(record)
data6 = Load_Data_1("csv/1menu.csv")
for i in data6:
record = Menu(**{'id': i[0], 'menu_name': i[1]})