def add_users():
users = {}
req = request.values
users['nama'] = req['nama'].upper()
users['username'] = req['username']
users['password'] = ph.hash(req['password'])
users['is_fb'] = str2bool(req['facebook'])
users['is_tw'] = str2bool(req['twitter'])
users['is_ig'] = str2bool(req['instagram'])
users['is_tg'] = str2bool(req['telegram'])
users['is_superadmin'] = False
users['tgl_post'] = datetime.today().strftime("%Y-%m-%d %H:%M:%S")
um.insert(users)
return jsonify(respon_action(200, "Success save data!"))
def update_role():
users = {}
req = request.values
id = req['id']
users['nama'] = req['nama'].upper()
users['username'] = req['username']
# users['password'] = ph.hash(req['password'])
users['is_fb'] = str2bool(req['facebook'])
users['is_tw'] = str2bool(req['twitter'])
users['is_ig'] = str2bool(req['instagram'])
users['is_tg'] = str2bool(req['telegram'])
# users['is_superadmin'] = False
users['tgl_post'] = datetime.today().strftime("%Y-%m-%d %H:%M:%S")
um.update({"_id": ObjectId(id)}, {"$set": users})
return jsonify(respon_action(200, "Success update data!"))
def __init__(self, rep, env, district, hh_id, hh_type, input_data, initial_action, la, lsoa):
self.rep = rep
self.rnd = self.rep.rnd
self.env = env
self.district = district
self.hh_id = hh_id
self.hh_type = hh_type
self.input_data = input_data
self.la = la
self.lsoa = lsoa
self.output_data = self.rep.output_data
self.initial_status = initial_action.type
self.digital = initial_action.digital
self.initial_time = initial_action.time
self.engaged = initial_action.engaged
self.district_name = district.district
self.priority = self.input_data['priority']
self.paper_allowed = h.str2bool(self.input_data['paper_allowed'])
self.paper_on_request = h.str2bool(self.input_data['paper_on_request'])
# flags to keep track of what the hh is doing/has done
if self.initial_time > 0 and (self.initial_status == 'early' or self.initial_status == 'late'):
self.resp_planned = True
else:
self.resp_planned = False
self.resp_time = 0
self.resp_level = 0
self.help_level = 0
self.responded = False
self.return_sent = False
self.return_received = False
self.status = ''
self.visits = 0
self.time_spent_visits = 0
self.calls = 0
self.arranged_visit = False
self.letter_count = 0
def __init__(self, rep, env, district, input_data, co_id):
self.rep = rep
self.env = env
self.district = district
self.input_data = input_data
self.co_id = co_id
self.rnd = self.rep.rnd
self.action_plan = []
self.start_date = dt.datetime.strptime((self.input_data['start_date']),
'%Y, %m, %d').date()
self.end_date = dt.datetime.strptime((self.input_data['end_date']),
'%Y, %m, %d').date()
self.has_pq = h.str2bool(self.input_data['has_pq'])
self.has_postcard = h.str2bool(self.input_data['has_postcard'])
self.start_sim_time = h.get_entity_time(
self) # self.co_start_time() # the sim time the co starts work
self.end_sim_time = h.get_entity_time(
self, "end") # self.co_end_time() # the sim time the co ends work
self.active_time = [] # as above but for active time
# start work at correct time
start_delayed(self.env, self.co_working_test(), self.start_sim_time)
def __init__(self, env, rep, district, input_data, letter_type):
self.env = env
self.rep = rep
self.district = district
self.input_data = input_data
self.letter_type = letter_type
self.blanket = h.str2bool(self.input_data["blanket"])
self.targets = self.input_data["targets"]
self.start_sim_time = h.get_event_time(self)
self.period = self.input_data["period"]
# add process to decide who to send letters too...but with a delay
start_delayed(self.env, self.fu_letter(), self.start_sim_time)
def __init__(self, rep, rnd, env, district, hh_id, hh_type, input_data, output_data):
self.rep = rep
self.rnd = rnd
self.env = env
self.district = district
self.hh_id = hh_id
self.hh_type = hh_type
self.input_data = input_data
self.output_data = output_data
self.digital = set_preference(self)
if self.digital:
self.delay = self.input_data['delay']['digital']
else:
self.delay = self.input_data['delay']['paper']
self.priority = self.input_data['priority']
self.paper_allowed = h.str2bool(self.input_data['paper_allowed'])
# flags to keep track of what the hh is doing/has done
self.resp_planned = False
self.responded = False
self.returned = False
self.resp_time = 0
self.status = ''
self.visits = 0
self.visit_times = []
self.calls = 0
self.arranged_visit = False
self.resp_level = 0
self.help_level = 0
# below define how the hh behaves depending on preference
self.resp_level = self.set_behaviour('response')
self.help_level = self.resp_level + self.set_behaviour('help')
#self.help_level = 0
self.rep.env.process(self.action())
def initial_action(self, input_data, first_interaction, hh_type, hh_geog,
hh_digital):
if hh_digital or h.str2bool(input_data['paper_allowed']):
# use default
behaviour = 'default'
else:
# use alt
behaviour = 'alt'
# set values to use
hh_resp = input_data['behaviours'][behaviour]['response']
# if call centre not in sim set call probability to zero even if input says otherwise
if self.rep.total_ad_instances > 0:
hh_help = input_data['behaviours'][behaviour]['help']
else:
hh_help = 0
response_test = self.rnd.uniform(
0, 100) # represents the COA to be taken.
# also at this point test if, if no barriers were in place, if they would be engaged or not...
if response_test <= input_data['behaviours']['default']['response']:
hh_eng = True
else:
hh_eng = False
if response_test <= hh_resp:
# respond but test when
return self.early_responder(input_data, hh_digital,
first_interaction, hh_type, hh_eng)
elif hh_resp < response_test <= hh_resp + hh_help:
# call for help return when
return self.help(input_data, hh_digital, first_interaction,
hh_type, hh_eng)
else:
# do nothing return 0 time
return oo.initial_action('do_nothing', hh_digital, 0, hh_eng)
def phone_call(self):
self.output_data['Call'].append(call(self.rep.reps,
self.district.input_data["LA"],
self.digital,
self.hh_type,
self.env.now))
self.calls += 1
if (not self.digital and not self.paper_allowed and
h.returns_to_date(self.district) < self.district.input_data['paper_trigger'] and
h.str2bool(self.input_data['paper_on_request'])):
# so provide paper if the conditions are met...
self.paper_allowed = True
censusv2.schedule_paper_drop(self, self, False)
yield self.env.timeout(0)
else:
yield self.env.process(self.phone_call_connect())
def modelSpecCNN(is_hybrid):
print(
"\n#############################\n#### Model Specification ####\n#############################\n"
)
batch_size = int(input("{:30}".format("Batch size:")))
num_epochs = int(input("{:30}".format("Number of epoch:")))
train_embedding_str = input(
"{:30}".format("Train embedding? (True/False):"))
train_embedding = helper.str2bool(train_embedding_str)
lr = float(input("{:30}".format("Learning rate:")))
if is_hybrid:
filter_sizes_str_word = input(
"{:30}".format("Word Filter sizes (e.g. 1,2,3):"))
num_filters_word = int(input(
"{:30}".format("Number of word filters:")))
filter_sizes_str_char = input(
"{:30}".format("Char Filter sizes (e.g. 1,2,3):"))
num_filters_char = int(input(
"{:30}".format("Number of char filters:")))
filter_sizes_word = [
int(elem) for elem in filter_sizes_str_word.split(',')
]
filter_sizes_char = [
int(elem) for elem in filter_sizes_str_char.split(',')
]
return batch_size, num_epochs, filter_sizes_word, num_filters_word, filter_sizes_char, num_filters_char, train_embedding, lr
else:
filter_sizes_str = input("{:30}".format("Filter sizes (e.g. 1,2,3):"))
num_filters = int(input("{:30}".format("Number of filters:")))
filter_sizes = [int(elem) for elem in filter_sizes_str.split(',')]
return batch_size, num_epochs, filter_sizes, num_filters, train_embedding, lr
def early_responder(self, input_data, digital, first_interaction, hh_type,
hh_eng):
# returns if the household will respond before any other interactions. "early" if yes "late" otherwise.
response_time = h.set_household_response_time(self.rep, input_data,
hh_type, digital)
if digital and response_time + input_data['delay'][
'digital'] <= first_interaction:
# add a counter to the district so we know how many hh have responded early
self.early_responders += 1
return oo.initial_action('early', digital, response_time, hh_eng)
elif not digital and h.str2bool(input_data['paper_allowed']) \
and response_time + input_data['delay']['paper'] <= first_interaction:
self.early_responders += 1
return oo.initial_action('early', digital, response_time, hh_eng)
else:
return oo.initial_action('late', digital, response_time, hh_eng)
def fu_visit_outcome(self, household):
household_returns = self.household_test(household, "success_rate")
if not household.return_sent and household_returns:
# hh have not responded yet and respond there and then either by paper or digital.
if oo.record_visit_success:
self.rep.output_data['Visit_success'].append(
oo.visit_output(self.rep.reps, self.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, household.visits,
self.env.now))
household.resp_planned = True
yield self.rep.env.process(
household.household_returns(household.calc_delay()))
time_worked = self.input_data['visit_times']['success'] / 60 + \
self.district.travel_dist / self.input_data["travel_speed"]
if oo.record_time_summary:
for key, value in self.rep.time_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += time_worked
for key, value in self.rep.time_totals.items():
value[str(getattr(household, key))] += time_worked
yield self.rep.env.timeout(time_worked)
elif (not household.return_sent and not household_returns
and h.responses_to_date(
self.district) < self.district.input_data['paper_trigger']
and household.visits == household.input_data['max_visits']
and h.str2bool(household.input_data['paper_after_max_visits'])
and not household.paper_allowed):
# hh have not responded but do not respond as a result of the visit.
# need extra here for when you fail but not at max visits?
if oo.record_visit_failed:
self.rep.output_data['Visit_failed'].append(
oo.generic_output(self.rep.reps,
household.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, self.env.now))
# leave paper in hope they respond?
household.paper_allowed = True
hq.schedule_paper_drop(household, 'Visit', 'pq', self.has_pq)
if oo.record_paper_summary:
# add to the summary of the amount of paper given
for key, value in self.rep.paper_summary.items():
value[str(getattr(household, key))][0] += 1
for key, value in self.rep.paper_totals.items():
value[str(getattr(household, key))] += 1
time_worked = self.input_data['visit_times']['failed'] / 60 + \
self.district.travel_dist / self.input_data["travel_speed"]
for key, value in self.rep.time_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += time_worked
for key, value in self.rep.time_totals.items():
value[str(getattr(household, key))] += time_worked
yield self.rep.env.timeout(time_worked)
elif not household.return_sent and not household_returns:
# failed but no max visits so do no more
if oo.record_visit_failed:
self.rep.output_data['Visit_failed'].append(
oo.generic_output(self.rep.reps,
household.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, self.env.now))
time_worked = self.input_data['visit_times']['failed'] / 60 + \
self.district.travel_dist / self.input_data["travel_speed"]
if oo.record_time_summary:
for key, value in self.rep.time_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += time_worked
for key, value in self.rep.time_totals.items():
value[str(getattr(household, key))] += time_worked
yield self.rep.env.timeout(time_worked)
def fu_visit_assist(self, household):
da_test = self.rnd.uniform(0, 100)
da_effectiveness = self.input_data['da_effectiveness'][
household.hh_type]
time_worked = self.input_data['visit_times']['query'] / 60
if oo.record_time_summary:
for key, value in self.rep.time_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += time_worked
for key, value in self.rep.time_totals.items():
value[str(getattr(household, key))] += time_worked
yield self.rep.env.timeout(time_worked)
# if digital, have already responded or can use paper skip straight to the outcome of the visit
if household.digital or household.return_sent or household.paper_allowed:
yield self.rep.env.process(self.fu_visit_outcome(household))
# if not digital try to persuade them to complete online.
elif not household.digital and da_test <= da_effectiveness:
time_worked = self.input_data['visit_times']['convert'] / 60
if oo.record_time_summary:
for key, value in self.rep.time_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += time_worked
for key, value in self.rep.time_totals.items():
value[str(getattr(household, key))] += time_worked
yield self.rep.env.timeout(time_worked)
household.digital = True
if oo.record_visit_convert:
self.rep.output_data['Visit_convert'].append(
oo.generic_output(self.rep.reps,
household.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, self.env.now))
yield self.rep.env.process(self.fu_visit_outcome(household))
# not digital, do not convince to go online but allowed to use paper so go to outcome
elif not household.digital and da_test > da_effectiveness and household.paper_allowed:
yield self.rep.env.process(self.fu_visit_outcome(household))
# if not digital, do not convince to complete online, and trigger and max visits not reached give paper if on.
elif (not household.digital and da_test > da_effectiveness
and h.responses_to_date(
self.district) < self.district.input_data['paper_trigger']
and household.visits == household.input_data['max_visits']
and h.str2bool(household.input_data['paper_after_max_visits'])):
household.paper_allowed = True
self.env.process(
hq.schedule_paper_drop(household, 'Visit', 'pq', self.has_pq))
time_worked = self.input_data['visit_times']['paper'] / 60 +\
self.district.travel_dist/self.input_data["travel_speed"]
if oo.record_time_summary:
for key, value in self.rep.time_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += time_worked
for key, value in self.rep.time_totals.items():
value[str(getattr(household, key))] += time_worked
yield self.rep.env.timeout(time_worked)
else:
# or suggest other forms of assistance to be decided...
# non implemented at present so another visit will be scheduled.
if oo.record_visit_assist:
self.rep.output_data['Visit_assist'].append(
oo.generic_output(self.rep.reps, self.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, self.env.now))
time_worked = self.input_data['visit_times']['paper'] / 60 + \
self.district.travel_dist / self.input_data["travel_speed"]
if oo.record_time_summary:
for key, value in self.rep.time_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += time_worked
for key, value in self.rep.time_totals.items():
value[str(getattr(household, key))] += time_worked
yield self.rep.env.timeout(time_worked)
def fu_visit_contact(self, household):
household.priority += 1 # automatically lower the priority of this hh after a visit
household.visits += 1
if oo.record_visit_summary:
for key, value in self.rep.visit_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += 1
for key, value in self.rep.visit_totals.items():
value[str(getattr(household, key))] += 1
if oo.record_visit:
self.rep.output_data['Visit'].append(
oo.visit_output(self.rep.reps, self.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, household.visits,
self.env.now))
if household.responded:
if oo.record_visit_wasted:
self.rep.output_data['Visit_wasted'].append(
oo.generic_output(self.rep.reps, self.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, self.env.now))
elif household.resp_planned:
if oo.record_visit_unnecessary:
self.rep.output_data['Visit_unnecessary'].append(
oo.generic_output(self.rep.reps, self.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, self.env.now))
household_is_in = self.household_test(household, "contact_rate")
if household_is_in:
if oo.record_visit_contact:
self.rep.output_data['Visit_contact'].append(
oo.generic_output(self.rep.reps, self.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, self.env.now))
# household.visits_contacted += 1
yield self.rep.env.process(self.fu_visit_assist(household))
elif (not household_is_in
and household.visits == household.input_data['max_visits']
and h.str2bool(household.input_data['paper_after_max_visits'])):
household.paper_allowed = True
hq.schedule_paper_drop(household, 'Visit', 'pq', self.has_pq)
time_worked = self.input_data["visit_times"]["out_paper"] / 60 +\
self.district.travel_dist / self.input_data["travel_speed"]
if oo.record_time_summary:
for key, value in self.rep.time_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += time_worked
for key, value in self.rep.time_totals.items():
value[str(getattr(household, key))] += time_worked
yield self.rep.env.timeout(time_worked)
else:
# out - add drop off of a note
if oo.record_visit_out:
self.rep.output_data['Visit_out'].append(
oo.generic_output(self.rep.reps, self.district.district,
household.la, household.lsoa,
household.digital, household.hh_type,
household.hh_id, self.env.now))
self.env.process(
hq.schedule_paper_drop(household, 'Visit', 'postcard',
self.has_postcard))
time_worked = self.input_data["visit_times"][
"out"] / 60 + self.district.travel_dist / self.input_data[
"travel_speed"]
if oo.record_time_summary:
for key, value in self.rep.time_summary.items():
value[str(getattr(household, key))][math.floor(
self.rep.env.now / 24)] += time_worked
for key, value in self.rep.time_totals.items():
value[str(getattr(household, key))] += time_worked
yield self.rep.env.timeout(time_worked)
def create_households(self):
list_of_hh_types = sorted(list(self.input_data['households'].keys()))
for hh_type in list_of_hh_types:
# get hh data for current type
hh_input_data = self.input_data['households'][hh_type]
for i in range(hh_input_data['number']):
# if allowed paper use different paper prop to if not...
if h.str2bool(hh_input_data['paper_allowed']):
paper_prop = hh_input_data['paper_prop_pf']
else:
paper_prop = hh_input_data['paper_prop_df']
# set if digital here?
hh_digital = h.set_preference(paper_prop, self.rnd)
# define where the hh is located
hh_geog = self.return_household_geog(
hh_input_data['cca_makeup'], hh_type, hh_digital)
self.total_households += 1
# determine initial HH action
hh_action = self.initial_action(hh_input_data,
self.first_interaction,
hh_type, hh_geog, hh_digital)
if hh_action.digital:
time_to_use = hh_action.time + hh_input_data['delay'][
'digital']
else:
time_to_use = hh_action.time + hh_input_data['delay'][
'paper']
if h.str2bool(hh_input_data['paper_allowed']
) and oo.record_paper_summary:
# add to the summary of the amount of paper given
for key, value in self.rep.paper_summary.items():
value[str(getattr(hh_geog, key))][0] += 1
for key, value in self.rep.paper_totals.items():
value[str(getattr(hh_geog, key))] += 1
if hh_action.type == 'early':
# don't need an instance of a household just directly record response/return at correct time
self.rep.total_responses += 1
self.total_responses += 1
if oo.record_responded:
self.rep.output_data['Return_sent'].append(
oo.generic_output(self.rep.reps, self.district,
hh_geog.la, hh_geog.lsoa,
hh_action.digital, hh_type,
self.rep.total_hh,
hh_action.time))
if oo.record_return_received:
self.rep.output_data['Return_received'].append(
oo.generic_output(self.rep.reps, self.district,
hh_geog.la, hh_geog.lsoa,
hh_action.digital, hh_type,
self.rep.total_hh, time_to_use))
# add household to summary of total responses
if oo.record_active_summary:
for key, value in self.rep.active_summary.items():
value[str(getattr(hh_geog, key))][math.floor(
time_to_use / 24)] += 1
for key, value in self.rep.active_totals.items():
value[str(getattr(hh_geog, key))] += 1
# if paper and early also need to record???
if oo.record_active_paper_summary and not hh_action.digital:
for key, value in self.rep.active_paper_summary.items(
):
value[str(getattr(hh_geog, key))][math.floor(
time_to_use / 24)] += 1
for key, value in self.rep.active_paper_totals.items():
value[str(getattr(hh_geog, key))] += 1
if oo.record_responded:
self.rep.output_data['Responded'].append(
oo.generic_output(self.rep.reps, self.district,
hh_geog.la, hh_geog.lsoa,
hh_action.digital, hh_type,
self.rep.total_hh, time_to_use))
else:
# create a household instance passing initial state
self.households.append(
household.Household(self.rep, self.env, self,
self.rep.total_hh, hh_type,
hh_input_data, hh_action,
hh_geog.la, hh_geog.lsoa))
# if self.rep.reps == 1:
if self.rep.reps > 0 and oo.record_hh_record:
self.rep.output_data['hh_record'].append(
oo.hh_record(self.rep.reps, self.district, hh_geog.la,
hh_geog.lsoa, hh_type, hh_action.type,
hh_action.digital,
hh_input_data['paper_allowed'],
hh_action.time))
if hh_action.type not in ['do_nothing', 'help'
] and oo.record_passive_summary:
for key, value in self.rep.passive_summary.items():
value[str(getattr(hh_geog, key))][math.floor(
time_to_use / 24)] += 1
for key, value in self.rep.passive_totals.items():
value[str(getattr(hh_geog, key))] += 1
self.rep.total_hh += 1
parser.add_argument("--feature_level",
type=helper.str2feature,
default='word')
parser.add_argument("--num_splits", type=int, default=10)
parser.add_argument("--use_glove", type=helper.str2bool, default=True)
args = vars(parser.parse_args())
path = os.path.dirname(os.path.abspath(__file__)) + "/../data/target/"
use_glove = args['use_glove']
if args["feature_level"] == "word":
use_ctxt_str = input(
"{:30}".format("Use context tweet? (True/False):"))
use_ctxt = helper.str2bool(use_ctxt_str)
batch_size, num_epochs, filter_sizes, num_filters, train_embedding, lr = modelSpecCNN(
False)
time_index = int(time.time())
for index in range(args['num_splits']):
layers.CNN_models.train_word_cnn(use_glove, use_ctxt, index,
batch_size, num_epochs,
filter_sizes, num_filters,
train_embedding, lr, time_index)
layers.CNN_models.test_word_cnn(use_glove, use_ctxt,
args['num_splits'], time_index)
elif args["feature_level"] == "char":
batch_size, num_epochs, filter_sizes, num_filters, train_embedding, lr = modelSpecCNN(
False)
time_index = int(time.time())
parser.add_argument('--dist', default='root_d', type=str)
parser.add_argument('--test_time', default='False', type=str)
parser.add_argument('--num_spatial_heads', default=4, type=int)
parser.add_argument('--orthogonal_loss_coef', default=0.01, type=float)
args = parser.parse_args()
conf.dataset_name = args.dataset_name
conf.hidden_size = args.hidden_size
conf.window_size = args.window_size
conf.lead_time = args.lead_time
conf.gl_reg_coef = args.gl_reg_coef
conf.dropout = args.dropout
conf.dist = args.dist
conf.num_spatial_heads = args.num_spatial_heads
conf.orthogonal_loss_coef = args.orthogonal_loss_coef
conf.group_lasso = helper.str2bool(args.group_lasso)
conf.test_time = helper.str2bool(args.test_time)
train_crime_local, train_crime_global, train_static, train_sample_indices, train_dist, train_y, valid_crime_local, valid_crime_global, valid_static, valid_sample_indices, valid_dist, valid_y, test_crime_local, test_crime_global, test_static, test_sample_indices, test_dist, test_y = data.readData(
conf.dataset_name, conf.window_size, conf.lead_time, conf.train_ratio,
conf.test_ratio, conf.dist, conf.time_unit)
print 'window_size:', conf.window_size, 'lead_time:', conf.lead_time, 'time_unit:', conf.time_unit, 'train_ratio:', conf.train_ratio, 'test_ratio:', conf.test_ratio, 'group_lasso:', conf.group_lasso, 'gl_reg_coef:', conf.gl_reg_coef, 'orthogonal_loss_coef:', conf.orthogonal_loss_coef, 'num_spatial_heads:', conf.num_spatial_heads
sys.stdout.flush()
if (conf.test_time == True):
train_crime_global = np.concatenate(
[train_crime_global, valid_crime_global], axis=0)
train_crime_local = np.concatenate(
[train_crime_local, valid_crime_local], axis=0)
train_static = np.concatenate([train_static, valid_static], axis=0)
