def test_holidays_for_year_zone_and_name(self):
d = SchoolHolidayDates()
res = d.holidays_for_year_zone_and_name(2017, 'A',
'Vacances de printemps')
self.assertEquals(len(res), 17)
for k, v in res.items():
self.assertEquals(sorted(v.keys()), self.EXPECTED_KEYS)
expected_dates = [
self.parse_date(date) for date in [
'2017-04-15', '2017-04-16', '2017-04-17', '2017-04-18',
'2017-04-19', '2017-04-20', '2017-04-21', '2017-04-22',
'2017-04-23', '2017-04-24', '2017-04-25', '2017-04-26',
'2017-04-27', '2017-04-28', '2017-04-29', '2017-04-30',
'2017-05-01'
]
]
self.assertEquals(sorted([v['date'] for v in res.values()]),
expected_dates)
with self.assertRaisesRegexp(UnsupportedYearException,
'No data for year: 2005'):
d.holidays_for_year_zone_and_name(2005, 'A',
'Vacances de printemps')
with self.assertRaisesRegexp(UnsupportedZoneException,
'Unsupported zone: D'):
d.holidays_for_year_zone_and_name(2017, 'D',
'Vacances de printemps')
with self.assertRaisesRegexp(UnsupportedHolidayException,
'Unknown holiday name: Foo'):
d.holidays_for_year_zone_and_name(2017, 'A', 'Foo')
def test_holiday_for_year_by_name(self):
d = SchoolHolidayDates()
res = d.holiday_for_year_by_name(2017, 'Vacances de la Toussaint')
self.assertEquals(len(res), 16)
for k, v in res.items():
self.assertEquals(sorted(v.keys()), self.EXPECTED_KEYS)
expected_dates = [
self.parse_date(date) for date in [
'2017-10-21', '2017-10-22', '2017-10-23', '2017-10-24',
'2017-10-25', '2017-10-26', '2017-10-27', '2017-10-28',
'2017-10-29', '2017-10-30', '2017-10-31', '2017-11-01',
'2017-11-02', '2017-11-03', '2017-11-04', '2017-11-05'
]
]
self.assertEquals(sorted([v['date'] for v in res.values()]),
expected_dates)
with self.assertRaisesRegexp(UnsupportedYearException,
'No data for year: 2005'):
self.assertEquals({},
d.holiday_for_year_by_name(
2005, 'Vacances de la Toussaint'))
with self.assertRaisesRegexp(UnsupportedHolidayException,
'Unknown holiday name: Foo'):
self.assertEquals({}, d.holiday_for_year_by_name(2017, 'Foo'))
def processing(dataInt):
## missing value
df = dataInt.copy()
df_num = df.drop(['timestamp','loc_1', 'loc_2', 'loc_secondary_1', 'loc_secondary_2', 'loc_secondary_3'], axis=1)
df_NonNum = df.select_dtypes(include=[np.object])
imputed_training_mice = mice(df_num.values)
data_mice = pd.DataFrame(imputed_training_mice, columns = df_num.columns, index = list(df.index.values))
dClean = data_mice.join(df_NonNum)
## drop variable inutile
d_tr = dClean.drop(['loc_1', 'loc_2', 'loc_secondary_1', 'loc_secondary_2', 'loc_secondary_3'], axis=1)
## create extra attribute
conv(d_tr)
d_tr['timestamp'] = pd.to_datetime(d_tr.timestamp, format = '%Y-%m-%dT%H:%M:%S.%f')
## create season and rangeInYear
s = pd.to_datetime(pd.Series(d_tr['timestamp']))
d_tr['rangeInYear'] = s.dt.strftime('%j').astype(int)
d_tr['season'] = d_tr['rangeInYear'].apply(lambda d : get_season(d))
#create jours working days
d_tr['is_business_day'] = d_tr['datetime_perso'].apply(lambda e : int(business_day(e)))
# Is it an holiday for zone A, B or C?
d = SchoolHolidayDates()
d_tr['is_holiday'] = d_tr['datetime_perso'].apply(lambda f : int(d.is_holiday(datetime.date(f))))
dataInt1 = d_tr.drop(['rangeInYear', 'datetime_perso', 'date', 'timestamp'], axis=1)
return (dataInt1)
def __add_holidays_columns__(self, df) :
d = SchoolHolidayDates()
new_df = df.copy()
new_df[cf.SCHOOL_HOLIDAYS_A_COL] = new_df[cf.DATE_COL].apply(lambda row : d.is_holiday_for_zone(date(row.year, row.month, row.day), 'A') )
new_df[cf.SCHOOL_HOLIDAYS_B_COL] = new_df[cf.DATE_COL].apply(lambda row : d.is_holiday_for_zone(date(row.year, row.month, row.day), 'B') )
new_df[cf.SCHOOL_HOLIDAYS_C_COL] = new_df[cf.DATE_COL].apply(lambda row : d.is_holiday_for_zone(date(row.year, row.month, row.day), 'C') )
return new_df
def progression(request):
date_rentree = reglage_date.objects.get(nom='Rentrée').jour
CB1 = reglage_date.objects.get(nom='Lundi concours blanc 1').jour
CB2 = reglage_date.objects.get(nom='Lundi concours blanc 2').jour
while date_rentree.weekday() != 0:
date_rentree += -datetime.timedelta(days=1)
print(date_rentree, date_rentree.weekday())
cours = []
tds = []
tps = []
semaines = []
d = SchoolHolidayDates()
for sean in seance.objects.all():
if sean.ressource.type_de_ressource()[0] == 'cours':
if sean.duree_seance == 0:
cours[-1].append(sean)
else:
for i in range(sean.duree_seance):
cours.append([sean])
if sean.ressource.type_de_ressource()[0] == 'td':
if sean.duree_seance == 0:
tds[-1].append(sean)
else:
for i in range(sean.duree_seance):
tds.append([sean])
if sean.ressource.type_de_ressource()[0] == 'tp':
if sean.duree_seance == 0:
tps[-1].append(sean)
else:
for i in range(sean.duree_seance):
tps.append([sean])
cours = cours + [''] * (max(len(cours), len(tds), len(tps)) - len(cours))
tds = tds + [''] * (max(len(cours), len(tds), len(tps)) - len(tds))
tps = tps + [''] * (max(len(cours), len(tds), len(tps)) - len(tps))
present = 0
i, j = 0, 0
print(tps)
while j < len(cours):
date = date_rentree + datetime.timedelta(days=7 * i)
if date < date.today():
color = "#cccdcd"
elif date <= date.today() and date > date.today() - datetime.timedelta(
days=7):
color = "#03a5f6"
else:
color = ""
if d.is_holiday_for_zone(date, 'C') and d.is_holiday_for_zone(
date + datetime.timedelta(days=1), 'C') and date.month != 8:
semaines.append([i + 1, date, color, False, True, False])
elif date == CB1 or date == CB2:
semaines.append([i + 1, date, color, False, False, True])
else:
semaines.append(
[i + 1, date, color, [cours[j], tds[j], tps[j]], False, False])
j += 1
i += 1
return render(request, 'progression.html', {'semaines': semaines})
def test_is_holiday(self):
d = SchoolHolidayDates()
self.assertTrue(d.is_holiday(datetime.date(2017, 12, 25)))
self.assertFalse(d.is_holiday(datetime.date(2017, 12, 1)))
with self.assertRaisesRegexp(UnsupportedYearException,
'No data for year: 2005'):
d.is_holiday(datetime.date(2005, 2, 7))
def test_supported_holidays_are_complete(self):
d = SchoolHolidayDates()
res = d.holidays_for_year(2019)
names = set()
for _, v in res.items():
names.add(v["nom_vacances"])
expected = set(SchoolHolidayDates.SUPPORTED_HOLIDAY_NAMES)
self.assertEquals(names, expected)
def test_holidays_for_year(self):
d = SchoolHolidayDates()
res = d.holidays_for_year(2018)
self.assertEquals(len(res), 151)
for k, v in res.items():
self.assertEquals(sorted(v.keys()), self.EXPECTED_KEYS)
with self.assertRaisesRegexp(UnsupportedYearException,
"No data for year: 2024"):
self.assertEquals({}, d.holidays_for_year(2024))
def test_is_holiday(self):
d = SchoolHolidayDates()
self.assertTrue(d.is_holiday(datetime.date(2017, 12, 25)))
self.assertFalse(d.is_holiday(datetime.date(2017, 12, 1)))
with self.assertRaisesRegexp(UnsupportedYearException,
"No data for year: 1985"):
d.is_holiday(datetime.date(1985, 2, 7))
with self.assertRaisesRegexp(ValueError,
"date should be a datetime.date"):
d.is_holiday(datetime.datetime(2017, 12, 1, 2, 0))
def test_is_holiday_for_zone(self):
d = SchoolHolidayDates()
self.assertTrue(d.is_holiday_for_zone(datetime.date(2009, 2, 7), "A"))
self.assertFalse(d.is_holiday_for_zone(datetime.date(2009, 2, 7), "B"))
self.assertFalse(d.is_holiday_for_zone(datetime.date(2009, 2, 7), "C"))
self.assertFalse(d.is_holiday_for_zone(datetime.date(2009, 3, 7), "A"))
self.assertFalse(d.is_holiday_for_zone(datetime.date(2009, 6, 7), "A"))
with self.assertRaisesRegexp(UnsupportedYearException,
"No data for year: 1985"):
d.is_holiday_for_zone(datetime.date(1985, 2, 7), "D")
with self.assertRaisesRegexp(UnsupportedZoneException,
"Unsupported zone: D"):
self.assertFalse(
d.is_holiday_for_zone(datetime.date(2009, 2, 7), "D"))
def test_is_holiday_for_zone(self):
d = SchoolHolidayDates()
self.assertTrue(d.is_holiday_for_zone(datetime.date(2009, 2, 7), 'A'))
self.assertFalse(d.is_holiday_for_zone(datetime.date(2009, 2, 7), 'B'))
self.assertFalse(d.is_holiday_for_zone(datetime.date(2009, 2, 7), 'C'))
self.assertFalse(d.is_holiday_for_zone(datetime.date(2009, 3, 7), 'A'))
self.assertFalse(d.is_holiday_for_zone(datetime.date(2009, 6, 7), 'A'))
with self.assertRaisesRegexp(UnsupportedYearException,
'No data for year: 2005'):
d.is_holiday_for_zone(datetime.date(2005, 2, 7), 'D')
with self.assertRaisesRegexp(UnsupportedZoneException,
'Unsupported zone: D'):
self.assertFalse(
d.is_holiday_for_zone(datetime.date(2009, 2, 7), 'D'))
def test_holidays_for_year_zone_and_name(self):
d = SchoolHolidayDates()
res = d.holidays_for_year_zone_and_name(2017, "A",
"Vacances de printemps")
self.assertEquals(len(res), 17)
for k, v in res.items():
self.assertEquals(sorted(v.keys()), self.EXPECTED_KEYS)
expected_dates = [
self.parse_date(date) for date in [
"2017-04-15",
"2017-04-16",
"2017-04-17",
"2017-04-18",
"2017-04-19",
"2017-04-20",
"2017-04-21",
"2017-04-22",
"2017-04-23",
"2017-04-24",
"2017-04-25",
"2017-04-26",
"2017-04-27",
"2017-04-28",
"2017-04-29",
"2017-04-30",
"2017-05-01",
]
]
self.assertEquals(sorted([v["date"] for v in res.values()]),
expected_dates)
with self.assertRaisesRegexp(UnsupportedYearException,
"No data for year: 1985"):
d.holidays_for_year_zone_and_name(1985, "A",
"Vacances de printemps")
with self.assertRaisesRegexp(UnsupportedZoneException,
"Unsupported zone: D"):
d.holidays_for_year_zone_and_name(2017, "D",
"Vacances de printemps")
with self.assertRaisesRegexp(UnsupportedHolidayException,
"Unknown holiday name: Foo"):
d.holidays_for_year_zone_and_name(2017, "A", "Foo")
def test_holidays_for_year_and_zone(self):
d = SchoolHolidayDates()
res = d.holidays_for_year_and_zone(2017, "A")
self.assertEquals(len(res), 118)
for k, v in res.items():
self.assertEquals(sorted(v.keys()), self.EXPECTED_KEYS)
self.assertTrue(v["vacances_zone_a"])
with self.assertRaisesRegexp(UnsupportedYearException,
"No data for year: 1985"):
self.assertFalse(d.holidays_for_year_and_zone(1985, "D"))
with self.assertRaisesRegexp(UnsupportedZoneException,
"Unsupported zone: D"):
self.assertFalse(d.holidays_for_year_and_zone(2017, "D"))
def test_holidays_for_year_and_zone(self):
d = SchoolHolidayDates()
res = d.holidays_for_year_and_zone(2017, 'A')
self.assertEquals(len(res), 120)
for k, v in res.items():
self.assertEquals(sorted(v.keys()), self.EXPECTED_KEYS)
self.assertTrue(v['vacances_zone_a'])
with self.assertRaisesRegexp(UnsupportedYearException,
'No data for year: 2005'):
self.assertFalse(d.holidays_for_year_and_zone(2005, 'D'))
with self.assertRaisesRegexp(UnsupportedZoneException,
'Unsupported zone: D'):
self.assertFalse(d.holidays_for_year_and_zone(2017, 'D'))
def get_encoded_calendar_df(data_final):
# Load school holidays for France
fr_holidays = SchoolHolidayDates()
df_vacances = pd.DataFrame()
for year in list(set(data_final['year'])):
df_vacances = pd.concat([df_vacances, pd.DataFrame.from_dict(fr_holidays.holidays_for_year(year)).T])
# Load bank holidays for France
df_jf = pd.DataFrame()
for year in list(set(data_final['year'])):
df_jf = pd.concat([df_jf, pd.DataFrame([
{'date': el[0], 'jour_ferie': el[1]} for el in sorted(holidays.FRA(years=year).items())])])
# Merge school and bank holidays
df_holidays = pd.merge(df_vacances, df_jf, how='outer', on='date')
# Create features from df_holidays dataframes (school holidays and bank holidays):
# - 3 binary features for school holidays, taking 1 if the given zone is on holiday, else 0 (vacances_zone_a,
# vacances_zone_b, vacances_zone_c)
# Definition of a dictionary to encode boolean into numeric
dict_map_vac = {
True: 1,
False: 0
}
# Apply dictionary to each holiday column for the three zones (A, B, C)
df_holidays['vacances_zone_a'] = df_holidays['vacances_zone_a'].map(dict_map_vac)
df_holidays['vacances_zone_b'] = df_holidays['vacances_zone_b'].map(dict_map_vac)
df_holidays['vacances_zone_c'] = df_holidays['vacances_zone_c'].map(dict_map_vac)
# - 1 binary feature for bank holiday, taking 1 if it is a bank holiday, else 0
# The column "jour ferie" contains either the name of the holiday or a missing value (NaN)
# The idea is to put a '1' when it's a holiday (i.e. when the value is different from nan, else 0)
df_holidays['jour_ferie'] = df_holidays['jour_ferie'].map(lambda x: 1 if str(x) != 'nan' else 0)
# - To go further: Try to create a combined feature with school and bank holidays
df_holidays['holiday'] = df_holidays['vacances_zone_a'] + df_holidays['vacances_zone_b'] + df_holidays[
'vacances_zone_c'] + df_holidays['jour_ferie']
df_holidays['date'] = df_holidays['date'].map(lambda x: str(x))
data_final_cal = pd.merge(data_final, df_holidays, how='left', left_on='release_date', right_on='date').fillna(0)
data_final_cal['month'] = data_final_cal['release_date'].map(lambda x: int(x[5:7]))
data_final_cal = apply_cos(data_final_cal, 'month', 'cos_month', 12)
return data_final_cal
def test_holiday_for_year_by_name(self):
d = SchoolHolidayDates()
res = d.holiday_for_year_by_name(2017, "Vacances de la Toussaint")
self.assertEquals(len(res), 16)
for k, v in res.items():
self.assertEquals(sorted(v.keys()), self.EXPECTED_KEYS)
expected_dates = [
self.parse_date(date) for date in [
"2017-10-21",
"2017-10-22",
"2017-10-23",
"2017-10-24",
"2017-10-25",
"2017-10-26",
"2017-10-27",
"2017-10-28",
"2017-10-29",
"2017-10-30",
"2017-10-31",
"2017-11-01",
"2017-11-02",
"2017-11-03",
"2017-11-04",
"2017-11-05",
]
]
self.assertEquals(sorted([v["date"] for v in res.values()]),
expected_dates)
with self.assertRaisesRegexp(UnsupportedYearException,
"No data for year: 1985"):
self.assertEquals({},
d.holiday_for_year_by_name(
1985, "Vacances de la Toussaint"))
with self.assertRaisesRegexp(UnsupportedHolidayException,
"Unknown holiday name: Foo"):
self.assertEquals({}, d.holiday_for_year_by_name(2017, "Foo"))
conv(d_tr)
#d_tr['day of week']=d_tr['datetime_perso'].dt.dayofweek
d_tr['timestamp'] = pd.to_datetime(d_tr.timestamp,
format='%Y-%m-%dT%H:%M:%S.%f')
## create season and rangeInYear
s = pd.to_datetime(pd.Series(d_tr['timestamp']))
d_tr['rangeInYear'] = s.dt.strftime('%j').astype(int)
## create jours working days
d_tr['is_business_day'] = d_tr['datetime_perso'].apply(
lambda e: int(business_day(e)))
# Is it an holiday for zone A, B or C?
d = SchoolHolidayDates()
d_tr['is_holiday'] = d_tr['datetime_perso'].apply(
lambda f: int(d.is_holiday(datetime.date(f))))
d_tr['season'] = d_tr['rangeInYear'].apply(lambda d: get_season(d))
d_tr = d_tr.drop(['rangeInYear', 'datetime_perso', 'date', 'timestamp'],
axis=1)
d_ready = pd.merge(d_tr, dataOut, on='ID', how='left')
## gerer les dummies et les variables num
train_new = d_ready[d_ready['consumption_1'].notnull()]
train_new = train_new.drop(['ID'], axis=1)
test_new = d_ready[d_ready['consumption_1'].isnull()]
test_new = test_new.drop(['consumption_1', 'consumption_2', 'ID'], axis=1)
train_new.shape
def home(request):
name = 'Home'
projects = Projet.objects.all().order_by('-created')
count = Projet.objects.all().count()
coffees = int(count * 12)
barre = BarreRaccourcie.objects.all()
cpu = int(psutil.cpu_percent())
disk_usage = int(psutil.disk_usage('/').percent)
cpu_freq = int(psutil.cpu_freq().current)
x = datetime.datetime.now()
year = int(x.strftime("%y"))
month = int(x.strftime("%m"))
day = int(x.strftime("%d"))
months = []
projects_months = []
onemonth = [1, 3, 5, 7, 8, 10, 12] #31 days month
zeromonth = [4, 6, 9, 11] #30 days month
strangemonth = 2 #28 or 29 days month (february)
for i in range(0, 7):
if month - (6 - i) < 1:
months.append(12 - (6 - month - i))
else:
months.append(month - (6 - i))
for project in projects:
projects_months.append(str(project.created)[5:7])
file = open("holidays.txt", "r+")
today = datetime.date(2000 + year, month, day)
date = file.readline()
if str(x)[:10] == str(date)[:10] and False:
is_holiday = file.readline()
daysbeforeholiday = int(file.readline())
else:
d = SchoolHolidayDates()
is_holiday = d.is_holiday_for_zone(
datetime.date(2000 + year, month, day), 'C')
daysbeforeholiday = 0
if is_holiday == False:
while is_holiday == False:
if month in onemonth and day == 31:
month = month + 1
day = 1
elif month in zeromonth and day == 30:
month = month + 1
day = 1
elif month == 2 and day == 28:
month = month + 1
day = 1
day = day + 1
is_holiday = d.is_holiday_for_zone(
datetime.date(2000 + year, month, day), 'C')
holiday_date = datetime.date(2000 + year, month, day)
delta = holiday_date - today
daysbeforeholiday = delta.days
is_holiday = False
open('holidays.txt', 'w').close()
file = open("holidays.txt", "r+")
file.write('%s\n' % datetime.datetime.now())
#file.write(datetime.datetime.now().strftime("%H:%M:%S") + os.linesep)
file.write('%s\n' % is_holiday)
file.write('%s\n' % daysbeforeholiday)
return render(
request, 'index.html', {
'projects': projects,
'count': count,
"barre": barre,
'cpu': cpu,
'disk': disk_usage,
'cpu_freq': cpu_freq,
'months': months,
'projects_months': projects_months,
'name': name,
'is_holiday': is_holiday,
'daysbeforeholiday': daysbeforeholiday,
'coffees': coffees
})
def __init__(self, filepath):
super(OperationsStatsTransformer, self).__init__(filepath)
self.bank_holidays = {}
self.school_holiday = SchoolHolidayDates()
def add_rows(data_df, new_data_df):
df = data_df.copy()
shd = SchoolHolidayDates()
df_tn = pd.read_csv(TOURN_FILE,
index_col=0,
parse_dates=True,
date_parser=parse_date,
sep=';')
for d in new_data_df.index:
print('Adding data for date:', d)
# print(new_data_df.index)
# Jour feries
fer = int(d.date() in JoursFeries.for_year(d.year).values())
# Vacances scolaires
vac = int(shd.is_holiday_for_zone(d.date(), 'C'))
# Tournament
tourn = 0
for tn_date in df_tn.index:
if d in pd.date_range(end=tn_date, periods=4):
tourn = 1
# Weather
t = d.isoformat()
url = f'https://api.darksky.net/forecast/{DARKSKY_API_KEY3}/48.854891,2.203673,{t}?exclude=hourly,currently&flags&lang=fr&units=si'
response = requests.get(url)
if response.status_code is not 200:
print('Http Error')
weather = response.json()
temp_max = get_attribute(weather, 'temperatureMax')
temp_min = get_attribute(weather, 'temperatureMin')
wind = get_attribute(weather, 'windGust')
rain = get_attribute(weather, 'precipProbability')
# add scale rain between 0 and 4 include
rain = add_scale_rain(rain)
# Competition a Saint-Cloud
nbPlay = new_data_df.loc[d, 'nbPlay']
# Restaurant open ?
open_res = new_data_df.loc[d, 'open']
# Number of covers
nbCov = new_data_df.loc[d, 'nbCov']
#print(nbCov)
#print(temp_min)
#print(type(wind))
df.loc[d] = pd.Series(
{
'fer': fer,
'vac': vac,
'tourn': tourn,
'tempMoy': round((temp_max + temp_min) / 2, 2),
'tempMax': temp_max,
'tempMin': temp_min,
'wind': wind,
'rain': rain, ##True if rain > 0.5 else False,
'nbPlay': nbPlay,
'open': open_res,
'nbCov': nbCov
},
name=d)
return df