def __init__(self, first_name, last_name, email, district, city):
LOGGER.info("Create a volunteer instance")
self.first_name = first_name
self.last_name = last_name
self.email = email
self.district = district
self.city = city
def _parse_action_parameters(self):
LOGGER.info("Realizando parse da ação no template...")
self.action_execution.parameters = {
k: self._change_value(v)
for k, v in self.action_execution.parameters.items()
}
def clean_raw(self, overwrite=False):
"""Basic cleaning and reducing of data"""
LOGGER.info(f"Cleaning NAIS file for month {self.month}...")
if not exists(self.csv_cleaned) or overwrite:
self.df_raw.clean_raw()
else:
LOGGER.info(f"NAIS file for month {self.month} has been cleaned.")
def fire_rules(execution: Execution):
if has_unstaged_files():
raise UnstagedFilesException()
if is_ahead():
raise BranchAheadException()
actions = []
for action_execution in execution.action.get("execution"):
do = action_execution.get("do")
action_from_template = do.get("action")
LOGGER.info(f"Validando método {action_from_template}...")
action_class = eval(capitalize_first_letter(action_from_template))
action = action_class(
ActionExecution(execution.variables, do.get("parameters"),
execution.arguments))
if not action.is_implemented:
LOGGER.warn("Método não implementado, favor revisar seu template!")
else:
actions.append(action)
# TODO implementar informativo quando a ação não puder ser executada por
# determinado motivo
for action in actions:
action.execute()
def check_password(self, password):
"""
Check if hashed password matches with actual password
"""
LOGGER.info("Check if the password is correct")
return check_password_hash(self.password_hash, password)
def process(self, overwrite=False):
"""Basic cleaning and reducing of data"""
LOGGER.info(f"Processing NAIS file for month {self.month}...")
if not exists(self.csv_processed) or overwrite:
self.df_clean.preprocess()
else:
LOGGER.info(
f"NAIS file for month {self.month} has been preprocessed.")
def pull(branch):
global GIT_REPO
LOGGER.info(f"Atualizando source {branch}...")
try:
GIT_REPO.git.pull()
except GitCommandError as e:
raise GitException(e.stdout)
def delete(pattern):
LOGGER.info(f"Excluindo source(es) {pattern}...")
branches = [
branch.replace(" ", "") for branch in git.branch().splitlines()
]
branches = list(filter(lambda x: x.startswith(pattern), branches))
[git.execute(["git", "source", "-D", branch]) for branch in branches]
def __init__(self, reference_id, action_name, organizing_institution,
address, district, city, description):
LOGGER.info("Create an action instance")
self.reference_id = reference_id
self.action_name = action_name
self.organizing_institution = organizing_institution
self.address = address
self.district = district
self.city = city
self.description = description
def is_ahead():
branch = retrieve_current_branch()
try:
commits_ahead = GIT_REPO.iter_commits(branch + ".." + branch)
number_of_commits = sum(1 for _ in commits_ahead)
return number_of_commits > 0
except GitCommandError:
LOGGER.info(f"Your current source ({branch}) doesn't exists on remote "
f"repo. Please use git push source {branch}.")
def get_photos_sync_interval(config):
sync_interval = DEFAULT_SYNC_INTERVAL_SEC
config_path = ["photos", "sync_interval"]
if not traverse_config_path(config=config, config_path=config_path):
LOGGER.warning(
f"sync_interval is not found in {config_path_to_string(config_path=config_path)}."
+ f" Using default sync_interval: {sync_interval} seconds ...")
else:
sync_interval = get_config_value(config=config,
config_path=config_path)
LOGGER.info(f"Syncing photos every {sync_interval} seconds.")
return sync_interval
def get_retry_login_interval(config):
retry_login_interval = DEFAULT_RETRY_LOGIN_INTERVAL_SEC
config_path = ["app", "credentials", "retry_login_interval"]
if not traverse_config_path(config=config, config_path=config_path):
LOGGER.warning(
f"retry_login_interval not found in {config_path_to_string(config_path=config_path)}."
+ f" Using default {retry_login_interval} seconds ...")
else:
retry_login_interval = get_config_value(config=config,
config_path=config_path)
LOGGER.info(f"Retrying login every {retry_login_interval} seconds.")
return retry_login_interval
def __init__(self,
first_name,
last_name,
email,
username,
password,
is_admin=False):
LOGGER.info("Create an user instance")
self.first_name = first_name
self.last_name = last_name
self.email = email
self.username = username
self.password_hash = generate_password_hash(password)
self.is_admin = is_admin
def checkout_new_branch(source: str, branch: str):
global GIT_REPO
checkout(source)
pull(source)
LOGGER.info(f"Criando source {branch} com base na source {source}...")
try:
_validate_existence(branch)
GIT_REPO.git.checkout(source, b=branch)
except GitCommandError as e:
raise GitException(e.stdout)
def download_file(item, local_file):
if not (item and local_file):
return False
LOGGER.info(f"Downloading {local_file} ...")
try:
with item.open(stream=True) as response:
with open(local_file, "wb") as file_out:
copyfileobj(response.raw, file_out)
item_modified_time = time.mktime(item.date_modified.timetuple())
os.utime(local_file, (item_modified_time, item_modified_time))
except (exceptions.ICloudPyAPIResponseException, FileNotFoundError,
Exception) as e:
LOGGER.error(f"Failed to download {local_file}: {str(e)}")
return False
return True
def remove_obsolete(destination_path, files):
removed_paths = set()
if not (destination_path and files is not None):
return removed_paths
for path in Path(destination_path).rglob("*"):
local_file = str(path.absolute())
if local_file not in files:
LOGGER.info(f"Removing {local_file} ...")
if path.is_file():
path.unlink(missing_ok=True)
removed_paths.add(local_file)
elif path.is_dir():
rmtree(local_file)
removed_paths.add(local_file)
return removed_paths
def merge(source: str, target: str, allow_merge_again: bool):
global GIT_REPO
if allow_merge_again or not _already_merged(target, source):
checkout(target)
pull(target)
LOGGER.info(
f"Realizando merge da source {source} com a source {target}...")
try:
GIT_REPO.git.merge(source, "--no-ff")
except GitCommandError:
raise GitException("O merge gerou conflitos, você precisa "
"resolvê-los antes de commitar!")
else:
LOGGER.warn(f"O merge da source {source} com a source {target} já "
f"foi realizado!")
class SolveQuizForm(FlaskForm):
"""
Form that show a question form to solve by the user
"""
LOGGER.info("Generate the question form for the quiz")
question = RadioField(choices=[], validators=[InputRequired()])
times_up = HiddenField("times_up")
submit = SubmitField("Next")
class UserForm(FlaskForm):
"""
Form for a user to edit its information
"""
LOGGER.info("Generate the user form")
fullname = StringField("Full Name", validators=[DataRequired()])
username = StringField("Username", validators=[DataRequired()])
email = StringField("E-Mail", validators=[Email()])
submit = SubmitField("Edit")
class UserForm(FlaskForm):
"""
Form for an admin to edit a user
"""
LOGGER.info("Generate the user form from admin")
fullname = StringField("Full name", validators=[DataRequired()])
username = StringField("Username", validators=[DataRequired()])
email = StringField("E-mail", validators=[DataRequired()])
is_admin = SelectField(u"Is admin?", choices=["True", "False"])
submit = SubmitField("Edit")
def send(config, last_send=None, dry_run=False):
sent_on = None
email = config_parser.get_smtp_email(config=config)
to_email = config_parser.get_smtp_to_email(config=config)
host = config_parser.get_smtp_host(config=config)
port = config_parser.get_smtp_port(config=config)
no_tls = config_parser.get_smtp_no_tls(config=config)
password = config_parser.get_smtp_password(config=config)
if last_send and last_send > datetime.datetime.now() - datetime.timedelta(hours=24):
LOGGER.info("Throttling email to once a day")
sent_on = last_send
elif email and host and port:
try:
sent_on = datetime.datetime.now()
if not dry_run:
smtp = smtplib.SMTP(host, port)
smtp.set_debuglevel(0)
smtp.connect(host, port)
if not no_tls:
smtp.starttls()
if password:
smtp.login(email, password)
msg = build_message(email)
smtp.sendmail(from_addr=email, to_addrs=to_email, msg=msg.as_string())
smtp.quit()
except (Exception) as e:
sent_on = None
LOGGER.error(f"Failed to send email: {str(e)}.")
else:
LOGGER.warning("Not sending 2FA notification because SMTP is not configured")
return sent_on
class SetForAQuizForm(FlaskForm):
"""
Form for a user to create a new quiz
"""
LOGGER.info("Generate the quiz form for the user")
number_of_questions = SelectField(
"Select the number of questions to solve",
choices=["5", "10", "15", "20"],
validators=[InputRequired()])
subject = SelectField("Select a subject",
choices=[],
validators=[InputRequired()])
submit = SubmitField("Generate the quiz")
class QuestionForm(FlaskForm):
"""
Form for an admin to add or edit a question
"""
LOGGER.info("Generate the question form from admin")
description = StringField("Problem", validators=[DataRequired()])
correct_answer = StringField("Correct Answer", validators=[DataRequired()])
subject = StringField("Subject related", validators=[DataRequired()])
false_answer_1 = StringField("Wrong Answer 1", validators=[DataRequired()])
false_answer_2 = StringField("Wrong Answer 2", validators=[DataRequired()])
false_answer_3 = StringField("Wrong Answer 3", validators=[DataRequired()])
levels = [(1, "Beginner"), (1.5, "Easy"), (2, "Normal"), (2.5, "Hard"),
(3, "Very Hard"), (5, "Fiendish")]
level = SelectField(u"Level",
choices=levels) # ToDo: Create a table for Levels
submit = SubmitField("Submit")
def main(args):
try:
px_print("\n-#- P H O E N I X B E G I N -#-\n")
LOGGER.info("Iniciando processamento...")
require_git_repo()
template = _get_template()
try:
validate(instance=template, schema=PHOENIX_SCHEMA)
LOGGER.info("Validando comando...")
command = _get_command(args=args, template=template)
LOGGER.info("Validando ação...")
action = _get_action(args=args, command=command)
variables = template.get("variables")
arguments = args[2:]
# Create execution object to carry template and
# arguments through execution
execution = Execution(
command=command,
action=action,
variables=variables,
arguments=arguments,
)
LOGGER.info("Executando regras...")
fire_rules(execution)
except ValidationError as e:
raise InvalidTemplateException(e.message)
except PhoenixWarningException as e:
LOGGER.warn(e.message)
except PhoenixException as e:
LOGGER.exception(e.message)
except Exception as e:
LOGGER.exception(str(e))
px_print("\n-#- P H O E N I X E N D -#-")
def sync():
last_send = None
enable_sync_drive = True
enable_sync_photos = True
drive_sync_interval = 0
photos_sync_interval = 0
sleep_for = 10
while True:
config = read_config()
username = config_parser.get_username(config=config)
if username:
try:
if ENV_ICLOUD_PASSWORD_KEY in os.environ:
password = os.environ.get(ENV_ICLOUD_PASSWORD_KEY)
utils.store_password_in_keyring(username=username,
password=password)
else:
password = utils.get_password_from_keyring(
username=username)
api = ICloudPyService(
apple_id=username,
password=password,
cookie_directory=DEFAULT_COOKIE_DIRECTORY,
)
if not api.requires_2sa:
if "drive" in config and enable_sync_drive:
sync_drive.sync_drive(config=config, drive=api.drive)
drive_sync_interval = config_parser.get_drive_sync_interval(
config=config)
if "photos" in config and enable_sync_photos:
sync_photos.sync_photos(config=config,
photos=api.photos)
photos_sync_interval = config_parser.get_photos_sync_interval(
config=config)
if "drive" not in config and "photos" not in config:
LOGGER.warning(
"Nothing to sync. Please add drive: and/or photos: section in config.yaml file."
)
else:
LOGGER.error("Error: 2FA is required. Please log in.")
# Retry again
sleep_for = config_parser.get_retry_login_interval(
config=config)
next_sync = (
datetime.datetime.now() +
datetime.timedelta(seconds=sleep_for)).strftime("%c")
LOGGER.info(f"Retrying login at {next_sync} ...")
last_send = notify.send(config, last_send)
sleep(sleep_for)
continue
except exceptions.ICloudPyNoStoredPasswordAvailableException:
LOGGER.error(
"Password is not stored in keyring. Please save the password in keyring."
)
sleep_for = config_parser.get_retry_login_interval(
config=config)
next_sync = (
datetime.datetime.now() +
datetime.timedelta(seconds=sleep_for)).strftime("%c")
LOGGER.info(f"Retrying login at {next_sync} ...")
last_send = notify.send(config, last_send)
sleep(sleep_for)
continue
if "drive" not in config and "photos" in config:
sleep_for = photos_sync_interval
enable_sync_drive = False
enable_sync_photos = True
elif "drive" in config and "photos" not in config:
sleep_for = drive_sync_interval
enable_sync_drive = True
enable_sync_photos = False
elif ("drive" in config and "photos" in config
and drive_sync_interval <= photos_sync_interval):
sleep_for = photos_sync_interval - drive_sync_interval
photos_sync_interval -= drive_sync_interval
enable_sync_drive = True
enable_sync_photos = False
else:
sleep_for = drive_sync_interval - photos_sync_interval
drive_sync_interval -= photos_sync_interval
enable_sync_drive = False
enable_sync_photos = True
next_sync = (datetime.datetime.now() +
datetime.timedelta(seconds=sleep_for)).strftime("%c")
LOGGER.info(f"Resyncing at {next_sync} ...")
if (config_parser.get_drive_sync_interval(config=config) < 0 if "drive"
in config else True and config_parser.get_photos_sync_interval(
config=config) < 0 if "photos" in config else True):
break
sleep(sleep_for)
def checkout(branch):
global GIT_REPO
LOGGER.info(f"Fazendo checkout da source {branch}...")
GIT_REPO.git.checkout(branch)
def load_user(user_id):
LOGGER.info("Set up an user loader")
return User.query.get(int(user_id))
def _log_quantiles(self):
accel = self.gdf["Acceleration"].quantile(np.linspace(0.05, 1, 9, 0), "lower")
alter = self.gdf["Alteration_Degrees"].quantile(
np.linspace(0.05, 1, 9, 0), "lower"
)
LOGGER.info(f"Quantiles: {accel}, {alter}")
