def get_next_node(): db = application.get_db_connection() all_nodes = [] [all_nodes.append(x.split(':')[1]) for x in db.keys(schema.HEARTBEAT_KEY.format(\ '*'))] if len(all_nodes) == 1: node = all_nodes[0] elif len(all_nodes) > 1: # check for lowest cpu load stat = {} for n in all_nodes: try: stats = json.loads(db.get(schema.HEARTBEAT_KEY.format(n))) if not stat: stat['node'] = n stat['load'] = stats['load'] else: # check 15 min load if stats['load'][2] < stat['load'][2]: stat['node'] = n stat['load'] = stats['load'] break except: pass # ignore errors node = stat['node'] else: node = None if node: return json.loads(db.get(schema.HEARTBEAT_KEY.format(node))) else: return None
def get_app_nodes(app_name=None): if not app_name: raise NameError('You must specify an app_name') db = application.get_db_connection() # HACK -- fix to not split(':') -- TODO: dynamically lookup node name return [get_node(x.split(':')[1]) for x in db.keys(schema.NODE_APPS_KEY.format('*')) \ if app_name in db.smembers(x)]
def delete_user(username=None): if not username: raise NameError('You must specify a username') db = application.get_db_connection() user_key = schema.USER_KEY.format(username) db.delete(user_key) return True
def remove_application_config(app=None): if not application: raise NameError('You must specify an application') db = application.get_db_connection() app_key = schema.APP_KEY.format(app) db.delete(app_key) return True
def delete_role(rolename=None): if not rolename: raise NameError('You must specify a rolename') db = application.get_db_connection() role_key = schema.ROLE_KEY.format(rolename) db.delete(role_key) return True
def delete_file(filename=None, version=-1, **kwargs): if not filename: raise NameError('You must specify a filename') db = application.get_db_connection() fs = gridfs.GridFS(db) return fs.delete( get_file(filename=filename, version=version, **kwargs)._id)
def update_application_config(app=None, config={}): if not application: raise NameError('You must specify an application') db = application.get_db_connection() app_key = schema.APP_KEY.format(app, '') db.set(app_key, json.dumps(config)) return True
def queue_daemon(app, rv_ttl=settings.TASK_QUEUE_KEY_TTL): db = application.get_db_connection() while True: msg = db.blpop(settings.TASK_QUEUE_NAME) print('Running task: {0}'.format(msg)) func, task_id, args, kwargs = pickle.loads(msg[1]) qkey = settings.TASK_QUEUE_NAME key = '{0}:{1}'.format(qkey, task_id) data = {'date': time.time(), 'task_id': task_id, 'status': 'running', 'result': None} db.set(key, json.dumps(data)) try: rv = func(*args, **kwargs) data['status'] = 'complete' except Exception, e: import traceback traceback.print_exc() rv = e data['status'] = 'error' if isinstance(rv, dict): rv = json.dumps(rv) else: rv = str(rv) data['result'] = rv if rv is not None: db.set(key, json.dumps(data)) db.expire(key, rv_ttl)
def create_role(rolename=None): if not rolename: raise NameError('You must specify a rolename') db = application.get_db_connection() role_key = schema.ROLE_KEY.format(rolename) data = schema.role(rolename) db.set(role_key, json.dumps(data)) return True
def create_role(rolename=None): if not rolename: raise NameError('You must specify a rolename') db = application.get_db_connection() if not list(db.roles.find({'rolename': rolename})): data = schema.role(rolename) db.roles.insert(data) return True
def save_file(filename=None, path=None, version=None, **kwargs): if not filename or not path: raise NameError('You must specify a filename and path') if not os.path.exists(path): raise RuntimeError('Unable to locate file {0}'.format(path)) db = application.get_db_connection() fs = gridfs.GridFS(db) return fs.put(open(path).read(), filename=filename, version=version, **kwargs)
def get_user(username=None, api_key=None): if not username and not api_key: raise NameError('You must specify a username or api_key') db = application.get_db_connection() if username: return db.users.find_one({'username': username}) else: return db.users.find_one({'api_key': api_key})
def delay(*args, **kwargs): db = application.get_db_connection() qkey = settings.TASK_QUEUE_NAME task_id = str(uuid.uuid4()) key = "{0}:{1}".format(qkey, task_id) s = pickle.dumps((f, task_id, args, kwargs)) db.rpush(settings.TASK_QUEUE_NAME, s) return DelayedResult(task_id)
def add_event(title=None, description=None, start_date=None, start_time=None, \ end_date=None, end_time=None, location=None, latlng=None, owner=None): if not title or not description or not location: raise NameError('You must specify a title, description, and location') db = application.get_db_connection() data = schema.event(title=title, description=description, start_date=start_date, \ start_time=start_time, end_date=end_date, end_time=end_time, \ location=location, latlng=latlng, owner=owner) return db.events.insert(data)
def publish_client_message(msg={}): if not msg: raise NameError('You must specify a message') db = application.get_db_connection() # check type if not isinstance(msg, dict): msg = {'data': msg} db.publish(settings.CLIENT_CHANNEL, json.dumps(msg)) return True
def get_node(node_name=None): if not node_name: raise NameError('You must specify a node_name') db = application.get_db_connection() node_hb = db.get(schema.HEARTBEAT_KEY.format(node_name)) if node_hb: data = json.loads(node_hb) else: data = None return data
def create_user(username=None, email=None, password=None, role=None, enabled=True): if not username or not password or not role: raise NameError('You must specify a username, password, and role') db = application.get_db_connection() user_key = schema.USER_KEY.format(username) data = schema.user(username=username, email=email, \ password=encrypt_password(password, settings.SECRET_KEY), \ role=role, enabled=enabled) db.set(user_key, json.dumps(data)) return True
def master_listener(): db = application.get_db_connection() ps = db.pubsub() # subscribe to master channel ps.subscribe(settings.MASTER_CHANNEL) # publish availability data = {'node': settings.NODE_NAME, 'status': 'available', 'version': settings.VERSION} utils.publish_client_message(data) for m in ps.listen(): print(m)
def client_listener(): log = config.get_logger('client_listener') db = get_db_connection() ps = db.pubsub() ps.subscribe(settings.CLIENT_CHANNEL) for m in ps.listen(): try: print(m) except Exception, e: log.error('Unable to parse client message: {0}'.format(e))
def get_application_config(app=None): if not application: raise NameError('You must specify an application') db = application.get_db_connection() app_key = schema.APP_KEY.format(app, '*') res = db.keys(app_key) if res: app = db.get(res[0]) else: app = None return app
def get_app_name(app_uuid=None): if not app_uuid: raise NameError('You must specify an app_uuid') db = application.get_db_connection() app = [x.split(':')[1] for x in db.keys(schema.APP_KEY.format('*', '*')) \ if json.loads(db.get(x))['uuid'] == app_uuid] if app: app = app[0] else: app = None return app
def release_application_port(port=None): if not port: raise NameError('You must specify a port') db = application.get_db_connection() ports = db.get(schema.PORTS_KEY) if ports: try: ports = json.loads(ports) ports.remove(port) db.set(schema.PORTS_KEY, json.dumps(ports)) except: pass
def create_user(username=None, email=None, password=None, role=None, enabled=True): if not username or not password or not role: raise NameError('You must specify a username, password, and role') db = application.get_db_connection() # check for existing if list(db.users.find({'username': username})): raise RuntimeError('An account with that username already exists') data = schema.user(username=username, email=email, \ password=encrypt_password(password, settings.SECRET_KEY), \ role=role, enabled=enabled) db.users.insert(data) return True
def get_application_config(app_name=None, app_uuid=None): if not app_name and not app_uuid: raise NameError('You must specify an app_name or app_uuid') db = application.get_db_connection() if not app_name: app_name = get_app_name(app_uuid) app_key = schema.APP_KEY.format(app_name, '*') res = db.keys(app_key) if res: app = json.loads(db.get(res[0])) else: app = None return app
def setUp(self): self.client = application.app.test_client() self.db = application.get_db_connection() salt = int(time.time()) self._test_user = '******'.format(salt) self._test_user_password = '******' self._test_user_email = '{0}@lumentica.com'.format(self._test_user) self._test_user_role = 'testrole_{0}'.format(salt) self._project_name = 'test_project_{0}'.format(salt) self._project_type = 'live' self._project_distro_id = 0 self._project_version = '1' self._project_arch = 'i386' self._project_packages = ['blender', 'gimp']
def heartbeat(): db = application.get_db_connection() k = schema.HEARTBEAT_KEY while True: data = { 'node': settings.NODE_NAME, 'address': settings.NODE_ADDRESS, 'port': settings.NODE_PORT, 'action': 'heartbeat', 'status': 'available', 'version': settings.VERSION, 'load': os.getloadavg(), } db.set(k, json.dumps(data)) db.expire(k, settings.HEARTBEAT_INTERVAL) time.sleep(settings.HEARTBEAT_INTERVAL)
def get_user(username=None, apikey=None): if not username and not apikey: raise NameError('You must specify a username or apikey') db = application.get_db_connection() if not username: data = [x.split(':')[1] for x in db.keys(schema.USER_KEY.format('*'))\ if json.loads(db.get(x))['apikey'] == apikey] if data: username = data[0] user_key = schema.USER_KEY.format(username) user = db.get(user_key) if user: data = json.loads(user) else: data = None return data
def reserve_application_port(port=None): if not port: raise NameError('You must specify a port') db = application.get_db_connection() ports = db.get(schema.PORTS_KEY) if not ports: ports = [] else: try: ports = json.loads(ports) except: ports = [] if port in ports: raise RuntimeError('Port already reserved') ports.append(port) db.set(schema.PORTS_KEY, json.dumps(ports)) return True
def toggle_user(username=None, enabled=None): if not username: raise NameError('You must specify a username') db = application.get_db_connection() user = db.users.find_one({'username': username}) if user: if enabled != None: user['enabled'] = enabled else: current_status = user['enabled'] if current_status: enabled = False else: enabled = True db.users.update({'username': username}, {"$set": {'enabled': enabled}}) return True else: raise RuntimeError('User not found')
def get_next_application_port(): db = application.get_db_connection() k = schema.PORTS_KEY port = None ports = db.get(k) if not ports: ports = [] else: try: ports = json.loads(ports) except: ports = [] # generate and make sure port not already used while True: port = Random().randint(settings.APP_MIN_PORT, settings.APP_MAX_PORT) if port not in ports: break return port
def create_application(name=None, owner=None, **kwargs): if not name or not owner: raise NameError('You must specify a name and owner') # check name for invalid chars for c in name: if c not in VALID_CHARS: raise ValueError('Invalid characters in name') db = application.get_db_connection() if get_application_config(name): raise NameError('An application by that name already exists') app_key = schema.APP_KEY.format(name, owner) app_data = schema.application(name=name, owner=owner) for k,v in kwargs.iteritems(): app_data[k] = v # uuid for app app_data['uuid'] = str(uuid.uuid4()) db.set(app_key, json.dumps(app_data)) return True
def toggle_user(username=None, enabled=None): if not username: raise NameError('You must specify a username') db = application.get_db_connection() user = db.users.find_one({'username': username}) if user: if enabled != None: user['enabled'] = enabled else: current_status = user['enabled'] if current_status: enabled = False else: enabled = True db.users.update({'username': username}, {"$set": {'enabled': enabled} }) return True else: raise RuntimeError('User not found')
def find_events(latlng=None): db = application.get_db_connection() if latlng: pass return db.events.find()
def delete_role(rolename=None): if not rolename: raise NameError('You must specify a rolename') db = application.get_db_connection() db.roles.remove(schema.role(rolename=rolename)) return True
def delete_user(username=None): if not username: raise NameError('You must specify a username') db = application.get_db_connection() db.users.remove({'username': username}) return True
def delete_event(uuid=None): if not uuid: raise NameError('You must specify a uuid') db = application.get_db_connection() return db.events.remove({'uuid': uuid})