def final_to_pca(in_path, out_path):
# action=td.read_im_action(in_path+"/")
action = utils.read_object(in_path)
out_path = out_path.replace(".final", ".pca")
pca = action.to_pca()
eigen = td.to_time_serie(pca)
img = td.to_img(eigen)
utils.save_img(out_path, img)
def main(args):
"""Executes training job."""
# Load data into a pandas DataFrame.
df = read_object(args.input_path, file_type="json").sample(frac=1.)
if args.max_data_rows:
df = df.iloc[:args.max_data_rows].copy()
print(f"Data contains {len(df)} rows")
# Preprocess and featurize data.
traindf, validdf = preprocess_data(df)
# Create data set for model input.
train_dataset = AmazonReviewDataset(traindf, args.model_name,
args.max_sequence_length)
valid_dataset = AmazonReviewDataset(validdf, args.model_name,
args.max_sequence_length)
print(train_dataset[0])
# Define model and trainer.
model = AutoModelForSequenceClassification.from_pretrained(
args.model_name, num_labels=2, torchscript=True)
training_args = TrainingArguments(
output_dir=os.path.join(args.model_dir, "results"),
num_train_epochs=args.epochs,
per_device_train_batch_size=args.train_batch_size,
per_device_eval_batch_size=args.valid_batch_size,
learning_rate=args.learning_rate,
adam_epsilon=args.adam_epsilon,
warmup_steps=500,
weight_decay=args.weight_decay,
logging_dir=os.path.join(args.model_dir, "logs"),
logging_steps=10,
evaluation_strategy="steps",
load_best_model_at_end=True)
trainer = Trainer(model=model,
args=training_args,
train_dataset=train_dataset,
eval_dataset=valid_dataset,
compute_metrics=compute_metrics)
# Train the model.
trainer.train()
trainer.evaluate()
# Save the model to the output_path defined in train_model.py.
device = torch.device("cuda")
dummy_row = train_dataset[0]
dummy_input = (dummy_row["input_ids"].unsqueeze(0).to(device),
dummy_row["attention_mask"].unsqueeze(0).to(device))
traced_model = torch.jit.trace(model, dummy_input)
torch.jit.save(traced_model, os.path.join(args.model_dir, "model.pth"))
def _run_event_loop(self):
while True:
canvas, green_player, blue_player = read_object(self.client)
pygame.surfarray.pixels3d(self.canvas)[:, :] = canvas
if green_player and blue_player:
pygame.draw.rect(
self.canvas, THECOLORS["green"],
pygame.Rect(green_player.x * 5, green_player.y * 5 + 60,
15, 15))
pygame.draw.rect(
self.canvas, THECOLORS["blue"],
pygame.Rect(blue_player.x * 5, blue_player.y * 5 + 60, 15,
15))
if pygame.event.get(
eventtype=QUIT) or pygame.key.get_pressed()[K_ESCAPE]:
return
self._process_keyboard_events()
pygame.display.flip()
self.clock.tick(30)
def read_composite(cls_path):
model=utils.read_object(cls_path)
rand=deep.RandomNum()
ae=autoencoder.init_autoencoder(model.ae_model,rand)
cls_nn=nn.init_nn(5,model.nn_model)
return FeatureExtractor(ae,cls_nn)
from utils import read_object, save_object
from main import PARTICIPANT
from hashing_engine import encrypt, decrypt
import datetime
import pytz
utc = pytz.timezone('UTC')
DATA_PATH = "data/participants.pkl"
participants = read_object(DATA_PATH)
user_email = input("Please enter the user email you want to delete: ")
users_hids = []
for participant in participants.values():
if participant.hashed_email == encrypt(
user_email) and participant.unsubscribe == False:
users_hids.append(participant.hashed_subject_id)
string_hids = f"\n".join([x.decode("utf-8") for x in users_hids])
print(
f'Found {len(users_hids)} ids subcribed with that email here is the list:\n{string_hids}'
)
user_to_unsubscribe = input(
"Type the item number of appearance corresponding to the sID /!\ start with 0, or type any letter to do thing: "
)
if user_to_unsubscribe.isdigit():
indice = int(user_to_unsubscribe)
def fetch_update_participants():
if os.path.exists(DATA_PATH):
participants = read_object(DATA_PATH)
else:
participants = {}
qualtrics.main(
logger
) #fetching the surveys from qualtrics and putting it in the TEMP_QUALTRICS_DATA path
surveys = pandas.read_csv(
TEMP_QUALTRICS_DATA
) # reading the downloaded survey from the TEMP_QUALTRICS_DATA
surveys = surveys.loc[2:] # only take data from row 2
surveys = surveys[surveys['Finished'] ==
'1'] # only consider surveys which are completed
surveys = surveys[surveys['Q1'] == "I agree."]
for index, response in surveys.iterrows():
esID = encrypt(response['id'])
if esID not in list(participants.keys()):
enrollment_date = datetime.datetime.strptime(
response['EndDate'], "%Y-%m-%d %H:%M:%S")
localtz = pytz.timezone(response.get(key='INATZ', default='UTC'))
participants[esID] = PARTICIPANT(
enrollment_date, esID, encrypt(response['Q21']),
response['INATZ'],
response.get(key='population', default='control'))
participant_df.loc[esID.decode("utf-8"),
str(enrollment_date.date())] = "ENROLLED"
else:
participant = participants[esID]
participants[esID] = PARTICIPANT(
participant.enrollment_date, participant.hashed_subject_id,
participant.hashed_email, participant.time_zone,
getattr(participant, "population", "test"))
refreshed_participant = participants[esID]
refreshed_participant.last_daily_date = participant.last_daily_date
refreshed_participant.last_weekly_date = participant.last_weekly_date
refreshed_participant.nb_sent_daily = participant.nb_sent_daily
refreshed_participant.nb_sent_weekly = participant.nb_sent_weekly
refreshed_participant.unsubscribe = getattr(
participant, "unsubscribe", False)
refreshed_participant.unsubscribe_dt = getattr(
participant, "unsubscribe_dt", None)
refreshed_participant.unsubscribe_email_sent = getattr(
participant, "unsubscribe_email_sent", False)
refreshed_participant.unsubscribe_email_dt = getattr(
participant, "unsubscribe_email_dt", None)
participant_df.loc[
esID.decode("utf-8"),
str(participant.enrollment_date.date())] = "ENROLLED"
# updating the participant object, in case (in the code) we actually update the participant object.
save_object(participants, DATA_PATH)
if os.path.exists(TEMP_QUALTRICS_DATA):
os.remove(TEMP_QUALTRICS_DATA)
else:
logger.error(
"Cannot delete qualtrics data because the file does not exits")
return participants
def make_action(raw_action_path): raw_action=utils.read_object(raw_action_path) p_clouds=[create_point_cloud(img) for img in raw_action.frames] return Action(p_clouds)
def feat_imgs(in_path, out_path):
action = utils.read_object(in_path)
sd = action.features()
sd_ts = td.to_time_serie(sd)
print(out_path)
td.visualize(out_path, sd_ts)
def skew_imgs(in_path, out_path):
action = utils.read_object(in_path)
skew = action.skew()
skew_ts = td.to_time_serie(skew)
print(out_path)
td.visualize(out_path, skew_ts)
def pcloud_to_img(in_path, out_path):
pcloud_action = utils.read_object(in_path)
out_path = out_path.replace(".cloud", ".img/")
print(out_path)
pcloud_action.save_projection(out_path)
def read_sda(sda_path,conf_path):
model=utils.read_object(sda_path)
free_vars=deep.LabeledImages()
hyper_params=tools.read_hyper_params(conf_path)
train,test,prob_dist=create_nn_fun(free_vars,model,hyper_params)
return deep.Classifier(free_vars,model,train,test,prob_dist)
def create_seq_dataset(action_dir,cls_path):
cls=utils.read_object(cls_path)
actions=get_actions(action_dir,cls)
def read_autoencoder(cls_path):
hyper_params=get_hyper_params()
model=utils.read_object(cls_path)
rand=deep.RandomNum()
return init_autoencoder(model,rand)
def __init__(self,path):
self.clusters=utils.read_object(path)