def api_for_command(command_name):
data = {'command': command_name, 'args': request.args.to_dict()}
logger.info(f'Received with [{data}]')
command_exists = command_name in commands_fns.keys()
if command_exists:
execute_command_task(data, None)
return jsonify({"success": command_exists})
def log_in(driver):
logger.info('Logging In')
# log in with Google
driver.find_element_by_xpath(
f'//text()[contains(.,"Continue with Google")]/ancestor::*[self::div][1]'
).click()
notion_window, google_window = tuple(driver.window_handles)
driver.switch_to_window(google_window)
time.sleep(0.75)
cookies.load_cookies(driver, 'google.plk')
driver.refresh()
existing_accounts = driver.find_elements_by_xpath(
'//div[@data-identifier="*****@*****.**"][@role="link"]')
if len(existing_accounts) > 0:
existing_accounts[0].click()
else:
# email input
find_element_and_send_keys(driver, '//input[@type="email"]',
os.getenv('GMAIL_EMAIL'))
find_element_and_click(driver, '//div[@role="button"]') # next button
# password input (https://nitratine.net/blog/post/encryption-and-decryption-in-python/)
pwd_path = urlparse(driver.current_url).path
f = Fernet(os.getenv('CYPHER_SECRET_KEY').encode())
decrypted_password = f.decrypt(
os.getenv('GMAIL_PSWD_ENCRYPTED').encode()).decode()
find_element_and_send_keys(driver, '//input[@type="password"]',
decrypted_password)
find_element_and_click(
driver, '//div[@role="button"][@id="passwordNext"]') # next button
wait = WebDriverWait(driver, 10)
wait.until(
lambda ctx_driver: urlparse(ctx_driver.current_url).path != pwd_path)
cookies.store_cookies(driver, 'google.plk')
wait.until(lambda ctx_driver: len(ctx_driver.window_handles) == 1)
driver.switch_to_window(notion_window)
wait.until(
lambda ctx_driver: ctx_driver.current_url != f'{notion_url}/login')
logger.info('Logged in')
def main():
logger.info("Execution Started!!!")
try:
train_data = pd.read_csv(
os.path.join(KAGGLE_RAW_DATA_DIR, "train.csv"), encoding="utf-8"
)
if SIMPLE_PROCESSING_TYPE:
logger.info("Performing simple text processing.")
train_data_simple = preprocess(train_data)
if type(train_data_simple) == pd.core.frame.DataFrame:
train_data_simple.to_csv(
os.path.join(SIMPLE_PROCESSED_DATA_DIR, "train_data_simple.csv"),
index=False,
encoding="utf-8",
)
else:
logger.error("Unable to write simple processed data!!!")
if COMPLEX_PROCESSING_TYPE:
logger.info("Performing complex text processing.")
train_data_complex = preprocess(train_data, preprocess_type="complex")
if type(train_data_complex) == pd.core.frame.DataFrame:
train_data_complex.to_csv(
os.path.join(COMPLEX_PROCESSED_DATA_DIR, "train_data_complex.csv"),
index=False,
encoding="utf-8",
)
else:
logger.error("Unable to write complex processed data!!!")
if VECTORIZE_DATA_SIMPLE:
vectorize_data(
os.path.join(SIMPLE_PROCESSED_DATA_DIR, "train_data_simple.csv"),
os.path.join(TRAIN_DATA_DIR_WI, "vectors_simple.csv"),
)
if VECTORIZE_DATA_COMPLEX:
vectorize_data(
os.path.join(COMPLEX_PROCESSED_DATA_DIR, "train_data_complex.csv"),
os.path.join(TRAIN_DATA_DIR_WI, "vectors_complex.csv"),
)
except Exception as e:
logger.error("Exception in main method : {}".format(str(e)))
return
logger.info("Execution successfully completed.")
def execute_command_task(event, context):
driver = selenium.get_driver()
driver.manage().window().maximize();
try:
fn = commands_fns[event['command']]
logger.info('Going to Notion.so')
enter_notion(driver)
logger.info(f'Enter command [{fn.__name__}]')
fn(driver, **event['args'])
logger.info(f'Finished command [{fn.__name__}]')
time.sleep(0.5)
except:
if current_env == 'linux':
for window in driver.window_handles:
driver.switch_to_window(window)
remote_path = f'{paths["screenshots"]}/{time.time()}-{fn.__name__}.png'
with s3fs.open(remote_path, 'wb') as remote_file:
with tempfile.NamedTemporaryFile('rb+', suffix='.png') as local_file:
driver.save_screenshot(local_file.name)
copy_file_data(local_file, remote_file)
logger.error(traceback.format_exc())
finally:
logger.info('Reached finally statement. Exiting program')
driver.quit()
def main():
logger.info("Execution Started!!!")
if RAW_DATA_INGEST:
logger.info("Getting data.")
ingest_status = get_data()
if not ingest_status:
logger.error(
"Execution abruptly stopped while creating raw dataset!!!")
return
try:
train_data = pd.read_csv(os.path.join(RAW_DATA_DIR, "train_data.csv"),
encoding="utf-8")
if SIMPLE_PROCESSING_TYPE:
logger.info("Performing simple text processing.")
train_data_simple = preprocess(train_data)
if type(train_data_simple) == pd.core.frame.DataFrame:
train_data_simple.to_csv(
os.path.join(SIMPLE_PROCESSED_DATA_DIR,
"train_data_simple.csv"),
index=False,
encoding="utf-8",
)
else:
logger.error("Unable to write simple processed data!!!")
return
if COMPLEX_PROCESSING_TYPE:
logger.info("Performing complex text processing.")
train_data_complex = preprocess(train_data,
preprocess_type="complex")
if type(train_data_complex) == pd.core.frame.DataFrame:
train_data_complex.to_csv(
os.path.join(COMPLEX_PROCESSED_DATA_DIR,
"train_data_complex.csv"),
index=False,
encoding="utf-8",
)
else:
logger.error("Unable to write complex processed data!!!")
return
if VECTORIZE_DATA_SIMPLE:
logger.info("Vectorizing simple processed data.")
if not vectorize_data(
os.path.join(SIMPLE_PROCESSED_DATA_DIR,
"train_data_simple.csv"),
"simple",
):
logger.error(
"Execution abruptly stopped while vectorizing simple data!!!"
)
return
if VECTORIZE_DATA_COMPLEX:
logger.info("Vectorizing complex processed data.")
if not vectorize_data(
os.path.join(COMPLEX_PROCESSED_DATA_DIR,
"train_data_complex.csv"),
"complex",
):
logger.error(
"Execution abruptly stopped while vectorizing complex data!!!"
)
return
if SIMPLE_DATA_CLASSIFICATION:
logger.info(
"Performing classification using simple processed data.")
if not perform_classification("simple"):
logger.error(
"Execution abruptly stopped while performing simple data classification!!!"
)
return
if COMPLEX_DATA_CLASSIFICATION:
logger.info(
"Performing classification using complex processed data.")
if not perform_classification("complex"):
logger.error(
"Execution abruptly stopped while performing complex data classification!!!"
)
return
if TEST_SET_EVALUATION:
logger.info("Evaluating test data.")
if not evaluate_test_data():
logger.error(
"Execution abruptly stopped while evaluating test data!!!")
return
except Exception as e:
logger.error("Exception in main method : {}".format(str(e)))
return
logger.info("Execution successfully completed.")
def main():
logger.info("Execution Started!!!")
if DOWNLOAD_RAW_DATA:
fetch_and_save_raw_data()
if CREATE_RAW_DATASET:
if not read_all_data():
logger.error(
"Execution abruptly stopped while creating raw dataset!!!")
return
try:
train_data = pd.read_csv(os.path.join(RAW_DATA_DIR,
"raw_train_data.csv"),
encoding="utf-8")
if SIMPLE_PROCESSING_TYPE:
logger.info("Performing simple text processing.")
train_data_simple = preprocess(train_data)
if type(train_data_simple) == pd.core.frame.DataFrame:
train_data_simple.to_csv(
os.path.join(SIMPLE_PROCESSED_DATA_DIR,
"train_data_simple.csv"),
index=False,
encoding="utf-8",
)
else:
logger.error("Unable to write simple processed data!!!")
return
if COMPLEX_PROCESSING_TYPE:
logger.info("Performing complex text processing.")
train_data_complex = preprocess(train_data,
preprocess_type="complex")
if type(train_data_complex) == pd.core.frame.DataFrame:
train_data_complex.to_csv(
os.path.join(COMPLEX_PROCESSED_DATA_DIR,
"train_data_complex.csv"),
index=False,
encoding="utf-8",
)
else:
logger.error("Unable to write complex processed data!!!")
return
if VECTORIZE_DATA_SIMPLE:
logger.info("Vectorizing simple processed data.")
if VECTORIZE_DATA_COMPLEX:
logger.info("Vectorizing complex processed data.")
if not vectorize_data(
os.path.join(COMPLEX_PROCESSED_DATA_DIR,
"train_data_complex.csv"),
"complex",
):
logger.error(
"Execution abruptly stopped while vectorizing complex data!!!"
)
return
except Exception as e:
logger.error("Exception in main method : {}".format(str(e)))
return
logger.info("Execution successfully completed.")
def read_all_data():
logger.info("Creating raw training and testing dataset.")
try:
kaggle_data = pd.read_csv(os.path.join(ORIG_DATA_DIR,
"train_data.csv"))
anger_0 = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "anger-ratings-0to1.train.txt"),
sep="\t",
header=None,
)[[1, 2]]
anger_1 = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "anger-ratings-0to1.dev.target.txt"),
sep="\t",
header=None,
)[[1, 2]]
fear_0 = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "fear-ratings-0to1.train.txt"),
sep="\t",
header=None,
)[[1, 2]]
fear_1 = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "fear-ratings-0to1.dev.gold.txt"),
sep="\t",
header=None,
)[[1, 2]]
joy_0 = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "joy-ratings-0to1.train.txt"),
sep="\t",
header=None,
)[[1, 2]]
joy_1 = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "joy-ratings-0to1.dev.gold.txt"),
sep="\t",
header=None,
)[[1, 2]]
sadness_0 = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "sadness-ratings-0to1.train.txt"),
sep="\t",
header=None,
)[[1, 2]]
sadness_1 = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "sadness-ratings-0to1.dev.gold.txt"),
sep="\t",
header=None,
)[[1, 2]]
kaggle_test_data = pd.read_csv(
os.path.join(ORIG_DATA_DIR, "test_data.csv"))
anger_test_data = pd.read_csv(
os.path.join(OTHERS_RAW_DATA,
"anger-ratings-0to1.test.target.txt"),
sep="\t",
header=None,
)[[1, 2]]
fear_test_data = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "fear-ratings-0to1.test.target.txt"),
sep="\t",
header=None,
)[[1, 2]]
joy_test_data = pd.read_csv(
os.path.join(OTHERS_RAW_DATA, "joy-ratings-0to1.test.target.txt"),
sep="\t",
header=None,
)[[1, 2]]
sadness_test_data = pd.read_csv(
os.path.join(OTHERS_RAW_DATA,
"sadness-ratings-0to1.test.target.txt"),
sep="\t",
header=None,
)[[1, 2]]
other_data = merge_dataset(
[
anger_0,
anger_1,
anger_test_data,
fear_0,
fear_1,
fear_test_data,
joy_0,
joy_1,
joy_test_data,
sadness_0,
sadness_1,
sadness_test_data,
],
[1],
)
other_data = other_data.rename(columns={1: "content", 2: "sentiment"})
other_data.loc[other_data["sentiment"] == "joy",
"sentiment"] = "happiness"
merged_data = merge_dataset([kaggle_data, other_data], "content")
lbl_encoder_obj, merged_data["type"] = perform_label_encoding(
merged_data["sentiment"])
if not save_objects(lbl_encoder_obj,
os.path.join(OBJECTS_DIR, "label_encoder.joblib")):
logger.error("Exception in saving label encoder object!!!")
return False
merged_data.drop(columns=["sentiment"], inplace=True)
train_data, test_data = stratified_split(merged_data, split_col="type")
save_csv_data(train_data,
os.path.join(RAW_DATA_DIR, "raw_train_data.csv"))
save_csv_data(test_data, os.path.join(RAW_DATA_DIR,
"raw_test_data.csv"))
save_csv_data(kaggle_test_data,
os.path.join(RAW_DATA_DIR, "raw_test_data_kaggle.csv"))
return True
except Exception as e:
logger.error(
"Exception in reading and writing all raw dataset : {}".format(
str(e)))
return False