def get_names():
person_name = 'not available'
employee_name = 'not available'
email_id = ''
list_personname = []
if not os.path.exists('./cropped_images'):
os.makedirs('./cropped_images')
filenames = listdir('./cropped_images/')
print(filenames)
if '.DS_Store' in filenames:
filenames.remove('.DS_Store')
for f in filenames:
unknown_image = face_recognition.load_image_file(os.path.join('./cropped_images/',f))
unknown_image_encoding = face_recognition.face_encodings(unknown_image)
if len(unknown_image_encoding)>0:
person_name = detect_face_names(unknown_image_encoding[0])
if person_name not in list_personname and person_name != 'not available':
list_personname.append(person_name)
print(person_name)
df = pd.read_excel('employee_database.xlsx')
if len(list_personname)!=0:
for i in range(df.shape[0]):
if list_personname[0] == df['EMP_ID'][i]:
email_id = df['Emp_emaiil_id'][i]
employee_name = df['Emp_name'][i]
if email_id!='':
send_email(email_id)
print("The name of the person is ",employee_name)
files = glob.glob('./cropped_images/*')
for f in files:
os.remove(f)
return employee_name
def train(model, epoch, layers):
"""Train the model."""
# Training dataset.
dataset_train = CrossarmDataset()
dataset_train.load_crossarm(glo_var.DATASET_DIR, "train")
dataset_train.prepare()
# Validation dataset
dataset_val = CrossarmDataset()
dataset_val.load_crossarm(glo_var.DATASET_DIR, "val")
dataset_val.prepare()
if layers == "heads":
print("Training network heads")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE,
epochs=epoch,
layers='heads')
else:
print("Training all layers")
model.train(dataset_train,
dataset_val,
learning_rate=config.LEARNING_RATE / 10,
epochs=epoch,
layers='all')
subject = "NEURAL NETWORK DONE TRAINING"
text = "YAY your neural work training is done"
email_notification.send_email(subject, text)
return None
def main():
row_count = get_db_row_count(database_name, "sites");
for x in range(1,row_count+1):
date_time = datetime.now()
site_details = get_site_details(database_name, x)
try:
last_update = date_time.strptime(site_details[7], "%Y-%m-%d").date()
except:
last_update = date.today() - timedelta(1)
if its_between_dawn_sunset(site_details[3]):
kwhr = get_energy_values(site_details[4], site_details[0], date_time.date())
write_energy_to_database(database_name, site_details[0], date_time, kwhr)
print("{}'s solar system produced {} KWhr of electricity so far".format(site_details[2], kwhr))
elif its_after_sunset(site_details[3]) and (last_update < date_time.date()):
kwhr = get_energy_values(site_details[4], site_details[0], date_time.date())
power_values = get_power_values(site_details[4], site_details[0], date_time.date(), date_time.date())
write_power_to_database(database_name, site_details[0], power_values)
touch_site(database_name,site_details[0],date_time.date())
if len(site_details[5]) > 0:
send_email(site_details[2], site_details[5], 'Solar production notification', kwhr)
print("{}'s solar system produced {} KWhr of electricity today".format(site_details[2], kwhr))
def run_scrapping(scrap_pages=600):
# run through all pages and get all the URLs of add details, save to CSV
get_details_url(scrap_pages)
directory = "car_update_data"
csv_file_name = "car_list_all_v2_sauto_update.csv"
if os.path.exists(directory):
car_list_all_v2_sauto_update = pd.DataFrame(columns=["car_brand", "car_model"])
print("creating new dataframe")
else:
os.makedirs(directory)
car_list_all_v2_sauto_update = pd.DataFrame(columns=["car_brand", "car_model"])
print("creating new dir and dataframe")
car_detail_url_list = pd.read_csv("car_detail_url_list.csv")
car_detail_url_list.reset_index(drop=True)
browser = run_browser()
for index, url in car_detail_url_list.iterrows():
if (index + 1) % 100 == 0:
print("restarting browser...")
browser.quit()
time.sleep(5)
browser = run_browser()
time.sleep(3)
try:
browser.get(url[0])
except Exception:
if "crash" in str(Exception.with_traceback):
print("browser has crashed")
browser.quit()
browser = run_browser()
browser.get(url[0])
time.sleep(5)
detail_df = scrape_car_detail(url[0], browser)
car_list_all_v2_sauto_update = car_list_all_v2_sauto_update.append(detail_df, sort=False)
print(f"details added: {index + 1}", end="\r", flush=True)
time.sleep(1)
if (index + 1) % 100 == 0:
car_list_all_v2_sauto_update.to_csv(f"{directory}/{csv_file_name}", index=False)
print("100 more details added, saving...")
if (index + 1) % 1000 == 0:
email_notification.send_email(f"{index + 1} details done")
car_list_all_v2_sauto_update.to_csv(f"{directory}/{csv_file_name}", index=False)
email_notification.send_email(f"got all the {index + 1} details")
def get_details_url(n_pages):
car_detail_url_list = []
index = 0
reload_browser = True
for n in range(1, (n_pages+1)):
if reload_browser:
browser = start_chrome(get_page_url(n), headless=True)
reload_browser = False
browser.get(get_page_url(n))
soup = bs(browser.page_source, "html.parser")
results = soup.find_all("a", class_= "toDetail")
all_urls = n_pages*15
for element in results:
url_part = element.get("href")
url = f"https://www.sauto.cz{url_part}"
if url not in car_detail_url_list:
car_detail_url_list.append(url)
index += 1
print(f"urls added: {index}/{all_urls}", end="\r", flush=True)
# save to csv each 500 URLs
if (n + 1) % 25 == 0:
df = pd.DataFrame(car_detail_url_list)
print(f"saving at {index}")
df.to_csv("car_detail_url_list.csv", index = False)
browser.quit()
reload_browser = True
time.sleep(2)
browser.quit()
df = pd.DataFrame(car_detail_url_list, columns = ["url"])
df.to_csv("car_detail_url_list.csv", index=False)
print(f"finished with {index} urls saved, time to finish {n_pages/50} hours")
time.sleep(20)
email_notification.send_email(f"got all {index} detail URLs")
email_notification.send_email(f"{index + 1} details done")
car_list_all_v2_sauto_update.to_csv(f"{directory}/{csv_file_name}", index=False)
email_notification.send_email(f"got all the {index + 1} details")
print("started script")
args = parser.parse_args()
try:
while True:
now = datetime.datetime.now()
weekday_now = now.weekday()
hour_now = now.hour
minute_now = now.minute
if hour_now == 8 or args.t:
if args.s:
print(f"run scrapping on {args.s} pages")
run_scrapping(scrap_pages = args.s)
print("updating model")
if args.u:
update_model(update_data=True)
else:
update_model(update_data=False)
if args.o:
break
time.sleep(60*50)
except Exception as er:
print(er)
email_notification.send_email(message=f"{er}")
# Start from ImageNet trained weights
weights_path = model.get_imagenet_weights()
else:
weights_path = args.weights
# Load weights
print("Loading weights ", weights_path)
if args.weights.lower() == "coco":
# Exclude the last layers because they require a matching
# number of classes
model.load_weights(weights_path,
by_name=True,
exclude=[
"mrcnn_class_logits", "mrcnn_bbox_fc",
"mrcnn_bbox", "mrcnn_mask"
])
else:
model.load_weights(weights_path, by_name=True)
# Train or evaluate
try:
train(model, args.epoch, args.layers)
except KeyboardInterrupt:
print("KeyboardInterrupt Detected")
except:
subject = "NEURAL NETWORK TRAINING FAILURE"
text = traceback.format_exc()
email_notification.send_email(subject, text)
print(text)
'''
Created on 16 kwi 2018
@author: Kacper
'''
from consumer import *
from flight_webscrap import check_flights
from email_notification import send_email
empty_model = MoneyModel(10)
for i in range(2):
empty_model.step()
check_flights(3, 6)
send_email("284")
print("end")