def get_next_node():
db = application.get_db_connection()
all_nodes = []
[all_nodes.append(x.split(':')[1]) for x in db.keys(schema.HEARTBEAT_KEY.format(\
'*'))]
if len(all_nodes) == 1:
node = all_nodes[0]
elif len(all_nodes) > 1:
# check for lowest cpu load
stat = {}
for n in all_nodes:
try:
stats = json.loads(db.get(schema.HEARTBEAT_KEY.format(n)))
if not stat:
stat['node'] = n
stat['load'] = stats['load']
else:
# check 15 min load
if stats['load'][2] < stat['load'][2]:
stat['node'] = n
stat['load'] = stats['load']
break
except:
pass # ignore errors
node = stat['node']
else:
node = None
if node:
return json.loads(db.get(schema.HEARTBEAT_KEY.format(node)))
else:
return None
def get_app_nodes(app_name=None):
if not app_name:
raise NameError('You must specify an app_name')
db = application.get_db_connection()
# HACK -- fix to not split(':') -- TODO: dynamically lookup node name
return [get_node(x.split(':')[1]) for x in db.keys(schema.NODE_APPS_KEY.format('*')) \
if app_name in db.smembers(x)]
def delete_user(username=None):
if not username:
raise NameError('You must specify a username')
db = application.get_db_connection()
user_key = schema.USER_KEY.format(username)
db.delete(user_key)
return True
def remove_application_config(app=None):
if not application:
raise NameError('You must specify an application')
db = application.get_db_connection()
app_key = schema.APP_KEY.format(app)
db.delete(app_key)
return True
def delete_role(rolename=None):
if not rolename:
raise NameError('You must specify a rolename')
db = application.get_db_connection()
role_key = schema.ROLE_KEY.format(rolename)
db.delete(role_key)
return True
def delete_file(filename=None, version=-1, **kwargs):
if not filename:
raise NameError('You must specify a filename')
db = application.get_db_connection()
fs = gridfs.GridFS(db)
return fs.delete(
get_file(filename=filename, version=version, **kwargs)._id)
def update_application_config(app=None, config={}):
if not application:
raise NameError('You must specify an application')
db = application.get_db_connection()
app_key = schema.APP_KEY.format(app, '')
db.set(app_key, json.dumps(config))
return True
def queue_daemon(app, rv_ttl=settings.TASK_QUEUE_KEY_TTL):
db = application.get_db_connection()
while True:
msg = db.blpop(settings.TASK_QUEUE_NAME)
print('Running task: {0}'.format(msg))
func, task_id, args, kwargs = pickle.loads(msg[1])
qkey = settings.TASK_QUEUE_NAME
key = '{0}:{1}'.format(qkey, task_id)
data = {'date': time.time(), 'task_id': task_id, 'status': 'running', 'result': None}
db.set(key, json.dumps(data))
try:
rv = func(*args, **kwargs)
data['status'] = 'complete'
except Exception, e:
import traceback
traceback.print_exc()
rv = e
data['status'] = 'error'
if isinstance(rv, dict):
rv = json.dumps(rv)
else:
rv = str(rv)
data['result'] = rv
if rv is not None:
db.set(key, json.dumps(data))
db.expire(key, rv_ttl)
def create_role(rolename=None):
if not rolename:
raise NameError('You must specify a rolename')
db = application.get_db_connection()
role_key = schema.ROLE_KEY.format(rolename)
data = schema.role(rolename)
db.set(role_key, json.dumps(data))
return True
def create_role(rolename=None):
if not rolename:
raise NameError('You must specify a rolename')
db = application.get_db_connection()
if not list(db.roles.find({'rolename': rolename})):
data = schema.role(rolename)
db.roles.insert(data)
return True
def save_file(filename=None, path=None, version=None, **kwargs):
if not filename or not path:
raise NameError('You must specify a filename and path')
if not os.path.exists(path):
raise RuntimeError('Unable to locate file {0}'.format(path))
db = application.get_db_connection()
fs = gridfs.GridFS(db)
return fs.put(open(path).read(), filename=filename, version=version, **kwargs)
def get_user(username=None, api_key=None):
if not username and not api_key:
raise NameError('You must specify a username or api_key')
db = application.get_db_connection()
if username:
return db.users.find_one({'username': username})
else:
return db.users.find_one({'api_key': api_key})
def create_role(rolename=None):
if not rolename:
raise NameError('You must specify a rolename')
db = application.get_db_connection()
if not list(db.roles.find({'rolename': rolename})):
data = schema.role(rolename)
db.roles.insert(data)
return True
def get_user(username=None, api_key=None):
if not username and not api_key:
raise NameError('You must specify a username or api_key')
db = application.get_db_connection()
if username:
return db.users.find_one({'username': username})
else:
return db.users.find_one({'api_key': api_key})
def delay(*args, **kwargs):
db = application.get_db_connection()
qkey = settings.TASK_QUEUE_NAME
task_id = str(uuid.uuid4())
key = "{0}:{1}".format(qkey, task_id)
s = pickle.dumps((f, task_id, args, kwargs))
db.rpush(settings.TASK_QUEUE_NAME, s)
return DelayedResult(task_id)
def add_event(title=None, description=None, start_date=None, start_time=None, \
end_date=None, end_time=None, location=None, latlng=None, owner=None):
if not title or not description or not location:
raise NameError('You must specify a title, description, and location')
db = application.get_db_connection()
data = schema.event(title=title, description=description, start_date=start_date, \
start_time=start_time, end_date=end_date, end_time=end_time, \
location=location, latlng=latlng, owner=owner)
return db.events.insert(data)
def publish_client_message(msg={}):
if not msg:
raise NameError('You must specify a message')
db = application.get_db_connection()
# check type
if not isinstance(msg, dict):
msg = {'data': msg}
db.publish(settings.CLIENT_CHANNEL, json.dumps(msg))
return True
def add_event(title=None, description=None, start_date=None, start_time=None, \
end_date=None, end_time=None, location=None, latlng=None, owner=None):
if not title or not description or not location:
raise NameError('You must specify a title, description, and location')
db = application.get_db_connection()
data = schema.event(title=title, description=description, start_date=start_date, \
start_time=start_time, end_date=end_date, end_time=end_time, \
location=location, latlng=latlng, owner=owner)
return db.events.insert(data)
def get_node(node_name=None):
if not node_name:
raise NameError('You must specify a node_name')
db = application.get_db_connection()
node_hb = db.get(schema.HEARTBEAT_KEY.format(node_name))
if node_hb:
data = json.loads(node_hb)
else:
data = None
return data
def create_user(username=None, email=None, password=None, role=None, enabled=True):
if not username or not password or not role:
raise NameError('You must specify a username, password, and role')
db = application.get_db_connection()
user_key = schema.USER_KEY.format(username)
data = schema.user(username=username, email=email, \
password=encrypt_password(password, settings.SECRET_KEY), \
role=role, enabled=enabled)
db.set(user_key, json.dumps(data))
return True
def master_listener():
db = application.get_db_connection()
ps = db.pubsub()
# subscribe to master channel
ps.subscribe(settings.MASTER_CHANNEL)
# publish availability
data = {'node': settings.NODE_NAME, 'status': 'available', 'version': settings.VERSION}
utils.publish_client_message(data)
for m in ps.listen():
print(m)
def client_listener():
log = config.get_logger('client_listener')
db = get_db_connection()
ps = db.pubsub()
ps.subscribe(settings.CLIENT_CHANNEL)
for m in ps.listen():
try:
print(m)
except Exception, e:
log.error('Unable to parse client message: {0}'.format(e))
def get_application_config(app=None):
if not application:
raise NameError('You must specify an application')
db = application.get_db_connection()
app_key = schema.APP_KEY.format(app, '*')
res = db.keys(app_key)
if res:
app = db.get(res[0])
else:
app = None
return app
def save_file(filename=None, path=None, version=None, **kwargs):
if not filename or not path:
raise NameError('You must specify a filename and path')
if not os.path.exists(path):
raise RuntimeError('Unable to locate file {0}'.format(path))
db = application.get_db_connection()
fs = gridfs.GridFS(db)
return fs.put(open(path).read(),
filename=filename,
version=version,
**kwargs)
def get_app_name(app_uuid=None):
if not app_uuid:
raise NameError('You must specify an app_uuid')
db = application.get_db_connection()
app = [x.split(':')[1] for x in db.keys(schema.APP_KEY.format('*', '*')) \
if json.loads(db.get(x))['uuid'] == app_uuid]
if app:
app = app[0]
else:
app = None
return app
def release_application_port(port=None):
if not port:
raise NameError('You must specify a port')
db = application.get_db_connection()
ports = db.get(schema.PORTS_KEY)
if ports:
try:
ports = json.loads(ports)
ports.remove(port)
db.set(schema.PORTS_KEY, json.dumps(ports))
except:
pass
def create_user(username=None, email=None, password=None, role=None, enabled=True):
if not username or not password or not role:
raise NameError('You must specify a username, password, and role')
db = application.get_db_connection()
# check for existing
if list(db.users.find({'username': username})):
raise RuntimeError('An account with that username already exists')
data = schema.user(username=username, email=email, \
password=encrypt_password(password, settings.SECRET_KEY), \
role=role, enabled=enabled)
db.users.insert(data)
return True
def get_application_config(app_name=None, app_uuid=None):
if not app_name and not app_uuid:
raise NameError('You must specify an app_name or app_uuid')
db = application.get_db_connection()
if not app_name:
app_name = get_app_name(app_uuid)
app_key = schema.APP_KEY.format(app_name, '*')
res = db.keys(app_key)
if res:
app = json.loads(db.get(res[0]))
else:
app = None
return app
def setUp(self):
self.client = application.app.test_client()
self.db = application.get_db_connection()
salt = int(time.time())
self._test_user = '******'.format(salt)
self._test_user_password = '******'
self._test_user_email = '{0}@lumentica.com'.format(self._test_user)
self._test_user_role = 'testrole_{0}'.format(salt)
self._project_name = 'test_project_{0}'.format(salt)
self._project_type = 'live'
self._project_distro_id = 0
self._project_version = '1'
self._project_arch = 'i386'
self._project_packages = ['blender', 'gimp']
def heartbeat():
db = application.get_db_connection()
k = schema.HEARTBEAT_KEY
while True:
data = {
'node': settings.NODE_NAME,
'address': settings.NODE_ADDRESS,
'port': settings.NODE_PORT,
'action': 'heartbeat',
'status': 'available',
'version': settings.VERSION,
'load': os.getloadavg(),
}
db.set(k, json.dumps(data))
db.expire(k, settings.HEARTBEAT_INTERVAL)
time.sleep(settings.HEARTBEAT_INTERVAL)
def get_user(username=None, apikey=None):
if not username and not apikey:
raise NameError('You must specify a username or apikey')
db = application.get_db_connection()
if not username:
data = [x.split(':')[1] for x in db.keys(schema.USER_KEY.format('*'))\
if json.loads(db.get(x))['apikey'] == apikey]
if data:
username = data[0]
user_key = schema.USER_KEY.format(username)
user = db.get(user_key)
if user:
data = json.loads(user)
else:
data = None
return data
def create_user(username=None,
email=None,
password=None,
role=None,
enabled=True):
if not username or not password or not role:
raise NameError('You must specify a username, password, and role')
db = application.get_db_connection()
# check for existing
if list(db.users.find({'username': username})):
raise RuntimeError('An account with that username already exists')
data = schema.user(username=username, email=email, \
password=encrypt_password(password, settings.SECRET_KEY), \
role=role, enabled=enabled)
db.users.insert(data)
return True
def reserve_application_port(port=None):
if not port:
raise NameError('You must specify a port')
db = application.get_db_connection()
ports = db.get(schema.PORTS_KEY)
if not ports:
ports = []
else:
try:
ports = json.loads(ports)
except:
ports = []
if port in ports:
raise RuntimeError('Port already reserved')
ports.append(port)
db.set(schema.PORTS_KEY, json.dumps(ports))
return True
def toggle_user(username=None, enabled=None):
if not username:
raise NameError('You must specify a username')
db = application.get_db_connection()
user = db.users.find_one({'username': username})
if user:
if enabled != None:
user['enabled'] = enabled
else:
current_status = user['enabled']
if current_status:
enabled = False
else:
enabled = True
db.users.update({'username': username}, {"$set": {'enabled': enabled}})
return True
else:
raise RuntimeError('User not found')
def get_next_application_port():
db = application.get_db_connection()
k = schema.PORTS_KEY
port = None
ports = db.get(k)
if not ports:
ports = []
else:
try:
ports = json.loads(ports)
except:
ports = []
# generate and make sure port not already used
while True:
port = Random().randint(settings.APP_MIN_PORT, settings.APP_MAX_PORT)
if port not in ports:
break
return port
def create_application(name=None, owner=None, **kwargs):
if not name or not owner:
raise NameError('You must specify a name and owner')
# check name for invalid chars
for c in name:
if c not in VALID_CHARS:
raise ValueError('Invalid characters in name')
db = application.get_db_connection()
if get_application_config(name):
raise NameError('An application by that name already exists')
app_key = schema.APP_KEY.format(name, owner)
app_data = schema.application(name=name, owner=owner)
for k,v in kwargs.iteritems():
app_data[k] = v
# uuid for app
app_data['uuid'] = str(uuid.uuid4())
db.set(app_key, json.dumps(app_data))
return True
def toggle_user(username=None, enabled=None):
if not username:
raise NameError('You must specify a username')
db = application.get_db_connection()
user = db.users.find_one({'username': username})
if user:
if enabled != None:
user['enabled'] = enabled
else:
current_status = user['enabled']
if current_status:
enabled = False
else:
enabled = True
db.users.update({'username': username}, {"$set": {'enabled': enabled} })
return True
else:
raise RuntimeError('User not found')
def find_events(latlng=None):
db = application.get_db_connection()
if latlng:
pass
return db.events.find()
def delete_role(rolename=None):
if not rolename:
raise NameError('You must specify a rolename')
db = application.get_db_connection()
db.roles.remove(schema.role(rolename=rolename))
return True
def delete_user(username=None):
if not username:
raise NameError('You must specify a username')
db = application.get_db_connection()
db.users.remove({'username': username})
return True
def delete_event(uuid=None):
if not uuid:
raise NameError('You must specify a uuid')
db = application.get_db_connection()
return db.events.remove({'uuid': uuid})
