def add_user_profile_response(user, formObj):
session = sblambda.get_db_session()
instances_update = """
WITH {formObj} AS form
MATCH (u:User {auth0_key:{user}})
CREATE (l:LeadForm)
SET
l.email=form.email,
l.company=form.company,
l.country=form.country,
l.industry=form.industry,
l.telephone=form.telephone,
l.jobrole=form.jobrole,
l.whyneo4j=form.whyneo4j,
l.submitted=timestamp()
CREATE (u)-[:SUBMITTED_FORM]->(l)
"""
results = session.run(instances_update,
parameters={
"user": user,
"formObj": formObj
})
for record in results:
return True
return False
def update_db(auth0_key, containersAndPorts):
session = sblambda.get_db_session()
instances_update = """
UNWIND {containers} AS c
WITH c
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)
WHERE
u.auth0_key = {auth0_key}
AND
s.taskid = c.taskArn
SET
s.ip = c.ip,
s.privip = c.privip,
s.port = c.port,
s.boltPort = c.boltPort,
s.backupPort = c.backupPort
"""
results = session.run(instances_update, parameters={"auth0_key": auth0_key, "containers": containersAndPorts}).consume()
if session.healthy:
session.close()
return results
def update_db(sandbox_hash_key, containersAndPorts):
session = sblambda.get_db_session()
instances_update = """
UNWIND {containers} AS c
WITH c
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)
WHERE
s.sandbox_hash_key = {sandbox_hash_key}
AND
s.taskid = c.taskArn
SET
s.ip = c.ip,
s.port = c.port,
s.boltPort = c.boltPort
"""
print('updating DB for sandbox hash key')
print(instances_update)
print(sandbox_hash_key)
print(containersAndPorts)
results = session.run(instances_update,
parameters={
"sandbox_hash_key": sandbox_hash_key,
"containers": containersAndPorts
})
if session.healthy:
session.close()
return results
def get_tasks_to_stop(sandboxHashkey):
session = sblambda.get_db_session()
instances_query = """
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)-[:IS_INSTANCE_OF]-(uc:Usecase)
WHERE
s.sandbox_hash_key = {sandbox_hash_key}
AND
s.running=True
RETURN
u.name AS name, s.taskid AS taskid, s.usecase AS usecase, s.ip AS ip, s.port AS port,
s.sandbox_hash_key AS sandboxHashkey,
s.boltPort AS boltPort,
id(s) AS sandboxId,
uc.model_image AS modelImage
"""
body = ""
statusCode = 200
contentType = 'text/plain'
results = session.run(instances_query,
parameters={"sandbox_hash_key": sandboxHashkey})
tasks = []
for record in results:
record = dict((el[0], el[1]) for el in record.items())
tasks.append(record["taskid"])
session.close()
return tasks
def update_user(id, auth0_key, profile):
session = sblambda.get_db_session()
instances_query = """
UNWIND {profile} AS pro
MATCH
(u:User)
WHERE
id(u)={id}
AND
u.auth0_key={auth0_key}
SET
u.auth0_last_ip = pro.last_ip
RETURN
id(u) AS id, u.name AS name, u.email AS email, u.auth0_key AS auth0_key
"""
results = session.run(instances_query, parameters={"id": id, "profile": profile, "auth0_key": auth0_key})
users = []
for record in results:
record = dict((el[0], el[1]) for el in record.items())
users.append(record)
return users
def get_emails_to_send_to_fc():
session = sblambda.get_db_session()
instances_query = """
MATCH
(u:User)
WHERE
NOT exists(u.resp_from_fc)
AND
(
NOT exists(u.sent_to_fc)
OR
u.sent_to_fc < (timestamp() - 1000 * 60 * 60 * 24 * 7)
)
SET
u.sent_to_fc = timestamp()
RETURN
id(u) AS id, u.name AS name, u.email AS email
"""
results = session.run(instances_query)
users = []
for record in results:
record = dict((el[0], el[1]) for el in record.items())
users.append(record)
if session.healthy:
session.close()
return users
def get_tasks_in_db_set():
session = sblambda.get_db_session()
## Find all instances and make sure they have IPs if available
instances_query = """
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)
WHERE
s.running=True
RETURN
u.name AS name, s.taskid AS taskid, s.usecase AS usecase, s.ip AS ip, s.privip AS privip, s.port AS port
"""
results = session.run(instances_query)
resultjson = []
tasks = []
allTasksInDb = set()
for record in results:
record = dict((el[0], el[1]) for el in record.items())
allTasksInDb.add(record["taskid"])
if not record["ip"] or not record["privip"]:
tasks.append(record["taskid"])
resultjson.append(record)
return {'allTasksInDb': allTasksInDb, 'tasksWithoutIp': tasks}
def lambda_handler(event, context):
body = ""
statusCode = 200
contentType = 'application/json'
tasks = get_tasks_in_db_set()
allTasksInDb = tasks['allTasksInDb']
tasksWithoutIp = tasks['tasksWithoutIp']
allTasksRunning = set(get_task_list())
expiredTasksInDb = get_expired_tasks_set()
print('TASKS IN DB, BUT NOT RUNNING')
print(allTasksInDb - allTasksRunning)
set_tasks_not_running(allTasksInDb - allTasksRunning)
print('TASKS RUNNING, BUT NOT IN DB')
print(allTasksRunning - allTasksInDb)
print("Calculated from - allTasksRunning:")
print(allTasksRunning)
print("Calculated from - allTasksInDb:")
print(allTasksInDb)
# This has a timing issue. TODO resolve and add functionality back,
# after we can ignore tasks which started in the last few minutes
#stop_tasks( allTasksRunning - allTasksInDb, 'Task running but not in DB' )
print('EXPIRED TASKS RUNNING')
print(expiredTasksInDb)
stop_tasks(expiredTasksInDb,
'Task running in DB but past expiration in DB')
print('MARKING TASKS FOR EMAIL REMINDER ONE')
mark_tasks_for_email()
if len(tasksWithoutIp) > 0:
taskInfo = get_task_info(list(tasksWithoutIp))
if len(taskInfo) > 0:
update_db(taskInfo.values())
# DISABLE - use CloudWatch
# check_utilization()
session = sblambda.get_db_session()
if session.healthy:
session.close()
return True
def mark_tasks_stopped(tasks):
session = sblambda.get_db_session()
instances_update = """
UNWIND {tasks} AS task
WITH task
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)
WHERE
s.taskid = task
SET
s.running = false
"""
results = session.run(instances_update, parameters={"tasks": tasks}).consume()
session.close()
return results
def updateTwitterUsername(auth0_key, auth_username):
session = sblambda.get_db_session()
update_query = """
MATCH
(u:User)
WHERE
u.auth0_key={auth0_key}
SET
u.auth_username={auth_username}
"""
session.run(update_query,
parameters={
"auth0_key": auth0_key,
"auth_username": auth_username
}).consume()
return True
def updateNameInfo(auth0_key, nameInfo):
session = sblambda.get_db_session()
update_query = """
MATCH
(u:User)
WHERE
u.auth0_key={auth0_key}
SET
u.fc_first_name={first_name},
u.fc_last_name={last_name}
"""
session.run(update_query,
parameters={
"auth0_key": auth0_key,
"first_name": nameInfo['nameDetails']['givenName'],
"last_name": nameInfo['nameDetails']['familyName']
}).consume()
def mark_tasks_for_email():
session = sblambda.get_db_session()
instances_update = """
MATCH
(s:Sandbox)
WHERE
s.running = True
AND
NOT EXISTS (s.sendEmailReminderOne)
AND
s.expires < (timestamp() + 1000 * 60 * 60 * 24 * 1)
SET
s.sendEmailReminderOne='Q'
"""
results = session.run(instances_update).consume()
return results
def updateIpInfo(auth0_key, ipInfo):
session = sblambda.get_db_session()
mmCountry = ipInfo['country']['names']['en']
if mmCountry == 'United States':
mmState = ipInfo['subdivisions'][0]['names']['en']
else:
mmState = ''
update_query = """
MATCH
(u:User)
WHERE
u.auth0_key={auth0_key}
SET
u.mm_country={mm_country},
u.mm_state={mm_state}
"""
session.run(update_query, parameters={"auth0_key": auth0_key, "mm_country": mmCountry, "mm_state": mmState}).consume()
def get_sandbox_to_backup(sandboxHashkey):
session = sblambda.get_db_session()
instances_query = """
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)-[:IS_INSTANCE_OF]-(uc:Usecase)
WHERE
s.sandbox_hash_key = {sandbox_hash_key}
AND
s.running=True
RETURN
s.ip AS ip, s.backupPort AS backupPort
"""
results = session.run(instances_query, parameters={"sandbox_hash_key": sandboxHashkey})
record = False
for record in results:
record = dict((el[0], el[1]) for el in record.items())
session.close()
return record
def update_connection_verified(sandbox_hash_key):
session = sblambda.get_db_session()
instances_update = """
MATCH
(s:Sandbox)
WHERE
s.sandbox_hash_key = {sandbox_hash_key}
SET
s.connection_verified = true
"""
print('updating verification for sandbox hash key')
results = session.run(instances_update,
parameters={"sandbox_hash_key": sandbox_hash_key})
if session.healthy:
session.close()
return results
def check_new_user(user, userIp, marketoCookie, utmSource):
session = sblambda.get_db_session()
user_query = """
MATCH (u:User {auth0_key:{user}})
WHERE
NOT EXISTS(u.lead_status)
AND
(timestamp() - (1000 * 60 * 1)) < u.createdAt
SET
u.lead_status='NEW',
u.lead_ip={userIp},
u.mm_cookie={marketoCookie},
u.utm_source={utmSource}
RETURN
u.email AS email,
u.lead_status AS lead_status
"""
results = session.run(user_query, parameters={"user": user, "userIp": userIp, "marketoCookie": marketoCookie, "utmSource": utmSource})
for record in results:
return record['lead_status']
return False
def update_db(containersAndPorts):
session = sblambda.get_db_session()
instances_update = """
UNWIND {containers} AS c
WITH c
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)
WHERE
s.taskid = c.taskArn
SET
s.ip = c.ip,
s.privip = c.privip,
s.port = c.port
"""
results = session.run(instances_update,
parameters={
"containers": containersAndPorts
}).consume()
return results
def set_tasks_not_running(tasks):
if len(tasks) > 0:
session = sblambda.get_db_session()
instances_update = """
UNWIND {tasks} AS t
WITH t
MATCH
(s:Sandbox)
WHERE
s.taskid = t
SET
s.running = False
"""
print("Tasks length: %s" % (len(tasks)))
results = session.run(instances_update,
parameters={
"tasks": list(tasks)
}).consume()
return results
else:
return False
def get_users_to_query_ip():
session = sblambda.get_db_session()
instances_query = """
MATCH
(u:User)
WHERE
NOT exists(u.mm_country)
AND
( exists(u.lead_ip) OR exists(u.auth0_last_ip) )
RETURN
id(u) AS id, u.name AS name, u.email AS email, u.auth0_key AS auth0_key, coalesce(u.lead_ip,u.auth0_last_ip) AS last_ip
"""
results = session.run(instances_query)
users = []
for record in results:
record = dict((el[0], el[1]) for el in record.items())
users.append(record)
return users
def get_users_to_query_twitter():
session = sblambda.get_db_session()
instances_query = """
MATCH
(u:User)
WHERE
u.auth0_key STARTS WITH 'twitter'
AND
NOT EXISTS (u.first_name)
AND
NOT EXISTS (u.auth_username)
RETURN
id(u) AS id, u.name AS name, u.email AS email, u.auth0_key AS auth0_key
"""
results = session.run(instances_query)
users = []
for record in results:
record = dict((el[0], el[1]) for el in record.items())
users.append(record)
return users
def get_users_to_query_names():
session = sblambda.get_db_session()
instances_query = """
MATCH
(u:User)
WHERE
(NOT EXISTS(u.first_name)
OR
NOT EXISTS (u.last_name))
AND
(NOT EXISTS(u.fc_first_name)
OR NOT EXISTS(u.fc_last_name))
RETURN u.auth0_key AS auth0_key, u.email AS email
"""
results = session.run(instances_query)
users = []
for record in results:
record = dict((el[0], el[1]) for el in record.items())
users.append(record)
return users
def grant_extension(user, days):
session = sblambda.get_db_session()
instances_update = """
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)
WHERE
u.auth0_key = {user}
AND
s.running = true
SET
s.expires = s.expires + (1000 * 60 * 60 * 24 * {days})
RETURN
s.sandbox_hash_key AS sandboxHashKey, s.expires AS expires
"""
results = session.run(instances_update,
parameters={
"user": user,
"days": days
})
for record in results:
return True
return False
def get_users_to_query_auth0():
session = sblambda.get_db_session()
instances_query = """
MATCH
(u:User)
WHERE
NOT exists(u.sent_to_auth0)
OR
u.sent_to_auth0 < (timestamp() - 1000 * 60 * 60 * 24 * 7)
SET
u.sent_to_auth0 = timestamp()
RETURN
id(u) AS id, u.name AS name, u.email AS email, u.auth0_key AS auth0_key
"""
results = session.run(instances_query)
users = []
for record in results:
record = dict((el[0], el[1]) for el in record.items())
users.append(record)
return users
def add_update_user(user, jwt, decodedjwt):
global REVALIDATE_USER
data = {"id_token": jwt}
profile = {}
name = '<default>'
first_name = None
last_name = None
email = '<default>'
picture = '<default>'
email = '<default>'
description = '<default>'
emailVerified = True
try:
req = urllib2.Request(url='%s%s' % (decodedjwt['iss'], 'tokeninfo'),
headers={"Content-type": "application/json"},
data=json.dumps(data))
f = urllib2.urlopen(url=req)
jsonProfile = f.read()
profile = json.loads(jsonProfile)
logger.debug(jsonProfile)
except urllib2.HTTPError as h:
if REVALIDATE_USER:
raise
else:
name = 'UNVALIDATED USER'
description = 'UNVALIDATED USER: SHOULD ONLY SEE IN TEST'
logger.warning(
'Allowing unvalidated user to proceed (REVALIDATE_USER=%s). User: "******"'
% (REVALIDATE_USER, user))
pass
if 'name' in profile:
name = profile['name']
if 'picture' in profile:
picture = profile['picture']
if 'description' in profile:
description = profile['description']
if 'email' in profile:
email = profile['email']
if 'email_verified' in profile:
if profile['email_verified'] == False:
emailVerified = False
if 'given_name' in profile:
first_name = profile['given_name']
elif 'user_metadata' in profile and 'given_name' in profile[
'user_metadata']:
first_name = profile['user_metadata']['given_name']
if 'family_name' in profile:
last_name = profile['family_name']
elif 'user_metadata' in profile and 'family_name' in profile[
'user_metadata']:
last_name = profile['user_metadata']['family_name']
query = """
MERGE
(u:User {auth0_key: {auth0_key}})
ON CREATE SET
u.name={name},
u.picture={picture},
u.description={description},
u.email={email},
u.email_verified={emailVerified},
u.lastAuthAt=timestamp(),
u.createdAt=timestamp(),
u.authCount=1
ON MATCH SET
u.picture={picture},
u.description={description},
u.email_verified={emailVerified},
u.lastAuthAt=timestamp(),
u.authCount = coalesce(u.authCount, 1) + 1
RETURN u
"""
session = sblambda.get_db_session()
session.run(query,
parameters={
"auth0_key": user,
"name": name,
"picture": picture,
"description": description,
"email": email,
"emailVerified": emailVerified
}).consume()
update_names_query = """
MATCH
(u:User {auth0_key: {auth0_key}})
SET
u.first_name={first_name},
u.last_name={last_name}
RETURN u
"""
if first_name or last_name:
session.run(update_names_query,
parameters={
"auth0_key": user,
"first_name": first_name,
"last_name": last_name
}).consume()
if session.healthy:
session.close()
return True
def lambda_handler(event, context):
sandbox_hash_key = event['queryStringParameters']['sandboxHashKey']
verifyConnect = False
if ('verifyConnect' in event['queryStringParameters']
and event['queryStringParameters']['verifyConnect'] == 'true'):
verifyConnect = True
session = sblambda.get_db_session()
instances_query = """
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)-[:IS_INSTANCE_OF]-(uc:Usecase)
WHERE
s.sandbox_hash_key = {sandbox_hash_key}
AND
s.running=True
RETURN
u.name AS name, s.taskid AS taskid, s.usecase AS usecase, s.ip AS ip, s.port AS port,
s.sandbox_hash_key AS sandboxHashKey,
s.boltPort AS boltPort,
id(s) AS sandboxId,
uc.model_image AS modelImage
"""
body = ""
statusCode = 200
contentType = 'text/plain'
results = session.run(instances_query,
parameters={"sandbox_hash_key": sandbox_hash_key})
tasks = []
for record in results:
record = dict((el[0], el[1]) for el in record.items())
if not (record["ip"] and record["boltPort"]):
tasks.append(record["taskid"])
if len(tasks) > 0:
taskInfo = get_task_info(tasks)
if len(taskInfo) > 0:
res = update_db(sandbox_hash_key, taskInfo.values())
for rec in res:
rec = dict((el[0], el[1]) for el in rec.items())
print("Result of update:")
print(rec)
# re-run DB query to get final data
results = session.run(instances_query,
parameters={"sandbox_hash_key": sandbox_hash_key})
tasks = []
foundRecord = False
for record in results:
foundRecord = True
record = dict((el[0], el[1]) for el in record.items())
if verifyConnect:
# verify we can connect
try:
s = socket.socket()
s.settimeout(5)
s.connect((record['ip'], int(record['boltPort'])))
s = socket.socket()
s.settimeout(5)
s.connect((record['ip'], int(record['port'])))
body = '%s:%s' % (record['ip'], record['boltPort'])
print('Successfully Connected')
update_connection_verified(sandbox_hash_key)
except:
body = '%s:%s' % (record['ip'], record['boltPort'])
print('Failed Connecting to: %s%s' %
(record['ip'], record['boltPort']))
statusCode = 404
else:
# no need to verify connection
if (record["ip"] and record["boltPort"]):
body = '%s:%s' % (record['ip'], record['boltPort'])
statusCode = 200
else:
results = session.run(
instances_query,
parameters={"sandbox_hash_key": sandbox_hash_key})
for record in results:
record = dict((el[0], el[1]) for el in record.items())
if (record["ip"] and record["boltPort"]):
body = '%s:%s' % (record['ip'], record['boltPort'])
statusCode = 200
else:
body = 'Found sandbox, but no ip/port'
statusCode = 404
if not foundRecord:
body = 'Sandbox not found'
statusCode = 404
if session.healthy:
session.close()
return {
"statusCode": statusCode,
"headers": {
"Content-type": contentType,
"Access-Control-Allow-Origin": "*"
},
"body": body
}
def lambda_handler(event, context):
auth0_key = event['requestContext']['authorizer']['principalId']
session = sblambda.get_db_session()
instances_query = """
MATCH
(u:User)-[:IS_ALLOCATED]-(s:Sandbox)-[:IS_INSTANCE_OF]-(uc:Usecase)
WHERE
u.auth0_key = {auth0_key}
AND
s.running=True
RETURN
u.name AS name, s.taskid AS taskid, s.usecase AS usecase, s.ip AS ip, s.privip AS privip, s.port AS port,
s.boltPort AS boltPort,
s.backupPort AS backupPort,
s.password AS password,
s.expires AS expires,
s.sandbox_hash_key AS sandboxHashKey,
s.connection_verified AS connectionVerified,
id(s) AS sandboxId,
uc.model_image AS modelImage
"""
body = ""
statusCode = 200
contentType = 'application/json'
results = session.run(instances_query, parameters={"auth0_key": auth0_key})
resultjson = []
tasks = []
for record in results:
record = dict((el[0], el[1]) for el in record.items())
if not (record["ip"] and record["privip"] and record["boltPort"] and record["backupPort"]):
tasks.append(record["taskid"])
else:
if record["password"]:
record["password"] = sblambda.decrypt_user_creds(record["password"])
resultjson.append(record)
if len(tasks) > 0:
taskInfo = get_task_info(tasks)
if len(taskInfo) > 0:
update_db(auth0_key, taskInfo.values())
# re-run DB query to get final data
results = session.run(instances_query, parameters={"auth0_key": auth0_key})
for record in results:
record = dict((el[0], el[1]) for el in record.items())
if record["password"]:
record["password"] = sblambda.decrypt_user_creds(record["password"])
resultjson.append(record)
# TODO update IPs from taskinfo
# TODO verify can connect to IP/port
body = json.dumps(resultjson)
if session.healthy:
session.close()
return { "statusCode": statusCode, "headers": { "Content-type": contentType, "Access-Control-Allow-Origin": "*" }, "body": body }
