def upload_data():
print('in upload_data()')
while(True):
print('uploading data...')
sleep_time=freq
try:
rds_connection=rds.RDS()
local_connection=rds.Local()
if(isinstance(rds_connection,rds.RDS) and isinstance(local_connection,rds.Local)):
ts_start=time.time()
#upload files to s3
s3_upload.upload_file_not_in_cloud()
#upload M2G entries to RDS
m2g.upload_missing_entries(rds_connection)
#upload EMA tables to RDS
ema_db.upoload_missing_data_ts(rds_connection,local_connection,'ema_data')
ema_db.upload_unuploaded_rows(rds_connection,local_connection,'reward_data')
ema_db.upload_unuploaded_rows(rds_connection,local_connection,'ema_storing_data')
#upload stats about data missing from the cloud
print('uploading missing data...')
missing_data.insert_missing_files_row(rds_connection)
missing_data.insert_missing_M2G(rds_connection)
missing_data.insert_missing_data(rds_connection,local_connection,'ema_data','missing_ema_data')
missing_data.insert_missing_data(rds_connection,local_connection,'ema_storing_data','missing_ema_storing_data')
missing_data.insert_missing_data(rds_connection,local_connection,'reward_data','missing_reward_data')
ts_end=time.time()
#elapsed time in minutes
elapsed=(ts_end-ts_start)
sleep_time=freq-elapsed
except:
print('Exception in controller')
if(sleep_time>60):
time.sleep(int(sleep_time))
def manage_heart_beat():
while(True):
print('hb')
try:
rds_connection=rds.RDS()
if(isinstance(rds_connection,rds.RDS)):
res=heart_beat.insert_heart_beat(rds_connection)
except:
print('Exception in manage_heart_beat() ')
time.sleep(heart_beat_freq)
def init(with_kvtable=True, with_predefined_configuration=True):
config.init(ctx, with_kvtable=with_kvtable, with_predefined_configuration=with_predefined_configuration)
Cfg.register({
"app.run_period,Stable" : {
"DefaultValue": "seconds=20",
"Format" : "Duration",
"Description" : """Period when the Main scheduling Lambda function is run.
The smaller, the more accurate and reactive is CloneSquad. The bigger, the cheaper is CloneSquad to run itself (Lambda executions,
Cloudwatch GetMetricData, DynamoDB queries...)
"""
},
"app.default_ttl" : "300",
"app.disable,Stable": {
"DefaultValue": 0,
"Format": "Bool",
"Description": """Flag to disable Main Lambda function responsible to start/stop EC2 instances.
It disables completly CloneSquad. While disabled, the Lambda will continue to be started every minute to test
if this flag changed its status and allow normal operation again."""
},
"app.archive_interact_events": "0"
})
log.debug("Setup management objects.")
log.debug("o_state setup...")
ctx["o_state"] = state.StateManager(ctx)
log.debug("o_ec2 setup...")
ctx["o_ec2"] = ec2.EC2(ctx, ctx["o_state"])
log.debug("o_ssm setup...")
ctx["o_ssm"] = ssm.SSM(ctx)
log.debug("o_targetgroup setup...")
ctx["o_targetgroup"] = targetgroup.ManagedTargetGroup(ctx, ctx["o_ec2"])
log.debug("o_cloudwatch setup...")
ctx["o_cloudwatch"] = cloudwatch.CloudWatch(ctx, ctx["o_ec2"])
log.debug("o_notify setup...")
ctx["o_notify"] = notify.NotifyMgr(ctx, ctx["o_state"], ctx["o_ec2"], ctx["o_targetgroup"], ctx["o_cloudwatch"])
log.debug("o_ec2_schedule setup...")
ctx["o_ec2_schedule"] = ec2_schedule.EC2_Schedule(ctx, ctx["o_ec2"], ctx["o_targetgroup"], ctx["o_cloudwatch"])
log.debug("o_scheduler setup...")
ctx["o_scheduler"] = scheduler.Scheduler(ctx, ctx["o_ec2"], ctx["o_cloudwatch"])
log.debug("o_interact setup...")
ctx["o_interact"] = interact.Interact(ctx)
log.debug("o_rds setup...")
ctx["o_rds"] = rds.RDS(ctx, ctx["o_state"], ctx["o_cloudwatch"])
log.debug("o_transferfamily setup...")
ctx["o_transferfamily"] = transferfamily.TransferFamily(ctx, ctx["o_state"], ctx["o_cloudwatch"])
def init(with_kvtable=True, with_predefined_configuration=True):
log.debug("Init.")
config.init(ctx, with_kvtable=with_kvtable, with_predefined_configuration=with_predefined_configuration)
Cfg.register({
"app.run_period,Stable" : {
"DefaultValue": "seconds=20",
"Format" : "Duration",
"Description" : """Period when the Main scheduling Lambda function is run.
The smaller, the more accurate and reactive is CloneSquad. The bigger, the cheaper is CloneSquad to run itself (Lambda executions,
Cloudwatch GetMetricData, DynamoDB queries...)
"""
},
"app.default_ttl" : "300",
"app.disable,Stable": {
"DefaultValue": 0,
"Format": "Bool",
"Description": """Flag to disable Main Lambda function responsible to start/stop EC2 instances.
It disables completly CloneSquad. While disabled, the Lambda will continue to be started every minute to test
if this flag changed its status and allow normal operation again."""
}
})
log.debug("Setup management objects.")
o_state = state.StateManager(ctx)
o_ec2 = ec2.EC2(ctx, o_state)
o_targetgroup = targetgroup.ManagedTargetGroup(ctx, o_ec2)
o_cloudwatch = cloudwatch.CloudWatch(ctx, o_ec2)
o_notify = notify.NotifyMgr(ctx, o_state, o_ec2, o_targetgroup, o_cloudwatch)
o_ec2_schedule = ec2_schedule.EC2_Schedule(ctx, o_ec2, o_targetgroup, o_cloudwatch)
o_scheduler = scheduler.Scheduler(ctx, o_ec2, o_cloudwatch)
o_interact = interact.Interact(ctx)
o_rds = rds.RDS(ctx, o_state, o_cloudwatch)
ctx.update({
"o_state" : o_state,
"o_ec2" : o_ec2,
"o_targetgroup" : o_targetgroup,
"o_cloudwatch" : o_cloudwatch,
"o_notify" : o_notify,
"o_ec2_schedule" : o_ec2_schedule,
"o_scheduler" : o_scheduler,
"o_interact" : o_interact,
"o_rds" : o_rds
})
import ema_db
import dep_data
import Log
import logging
import threading
import time
import concurrent.futures
from concurrent.futures import ThreadPoolExecutor
import rds
import missing_data
import heart_beat
import threading
#connect to local and cloud(RDS) databases
try:
rds_connection=rds.RDS()
local_connection=rds.Local()
if(isinstance(rds_connection,rds.RDS) and isinstance(local_connection,rds.Local)):
#do these tasks just at the start of the deployment
ema_db.upload_fixed_tables(local_connection,rds_connection)
dep_data.upload_dep_data_table(rds_connection)
except:
print('cannot upload fixed data. Exception occured')
#how frequent we upload data (in seconds)
freq=10*60
def upload_data():
print('in upload_data()')
while(True):
print('uploading data...')