def CrawlerByDate(startDate: datetime, endDate: datetime):
days_between = endDate.__sub__(startDate).days
if (not days_between > 0):
raise Exception(
"Dates in wrong format or order. Should have at least one day between the dates."
)
with ByDateDatasetBuilder() as persister:
for days in range(0, days_between):
query_date = startDate + timedelta(days=days)
concepa = Concepa()
response = concepa.GetByDate(302, query_date)
persister.Save(response.ToCsv())
def load_weather(self, date_from: datetime, date_to: datetime,
city: City) -> List[str]:
session = create_session()
data = []
url = city.link
winds = session.query(Winds).filter(
and_(Winds.date >= date_from.strftime('%Y-%m-%d'),
Winds.date <= date_to.strftime('%Y-%m-%d'),
Winds.city_id == city.id)).all()
if len(winds) == 0:
days = date_to.__sub__(date_from).days
for i in range(1, days):
date = date_to - timedelta(days=i)
direction = self.weather.get_wind_direction(url, date)
if not direction:
continue
data.append(direction)
wind = Winds()
wind.date = date
wind.city_id = city.id
wind.wind = direction
session.add(wind)
session.commit()
else:
data = list(map(lambda x: x.wind, winds))
db_date_from = min(list(map(lambda x: x.date, winds)))
days = db_date_from.__sub__(date_from).days
for i in range(1, days):
date = db_date_from - timedelta(days=i)
direction = self.weather.get_wind_direction(url, date)
if not direction:
continue
data.append(direction)
wind = Winds()
wind.date = date
wind.city_id = city.id
wind.wind = direction
session.add(wind)
session.commit()
return data
