Exemplo n.º 1
0
 def set(self, cursor=None, user_id=None, staff_user_id=None, venue_id=None,
         manager=None, promo_perm=None, delete=None, **kwargs):
     if util.to_bool(delete):
         qry = {'delete':   'venue_staff',
                'where':    ('user_id = ?', 'venue_id = ?')}
         cursor.execute(util.query(**qry), (staff_user_id, venue_id))
         qry = {'delete':   'venue_managers',
                'where':    ('user_id = ?', 'venue_id = ?')}
         cursor.execute(util.query(**qry), (staff_user_id, venue_id))
     elif util.to_bool(manager):
         qry = {'select':   'id',
                'table':    'venue_managers',
                'where':    ('user_id = ?', 'venue_id = ?'),
                'order_by': 'id',
                'limit':     1}
         cursor.execute(util.query(**qry), (staff_user_id, venue_id))
         res = cursor.fetchone()
         if not res:
             qry = {'delete':   'venue_staff',
                    'where':    ('user_id = ?', 'venue_id = ?')}
             cursor.execute(util.query(**qry), (staff_user_id, venue_id))
             qry = {'insert_into': 'venue_managers',
                    'columns':     ('user_id', 'venue_id', 'time')}
             cursor.execute(util.query(**qry), (staff_user_id, venue_id,
                                                util.now()))
     else:
         qry = {'select':   'id',
                'table':    'venue_staff',
                'where':    ('user_id = ?', 'venue_id = ?'),
                'order_by': 'id',
                'limit':     1}
         cursor.execute(util.query(**qry), (staff_user_id, venue_id))
         res = cursor.fetchone()
         if not res:
             qry = {'delete':   'venue_managers',
                    'where':    ('user_id = ?', 'venue_id = ?')}
             cursor.execute(util.query(**qry), (staff_user_id, venue_id))
             qry = {'insert_into': 'venue_staff',
                    'columns':     ('user_id', 'venue_id', 'time',
                                    'promo_perm')}
             cursor.execute(util.query(**qry), (staff_user_id, venue_id,
                                                util.now(),
                                                1 if util.to_bool(promo_perm) else 0))
         else:
             qry = {'update':     'venue_staff',
                    'set_values': ('promo_perm'),
                    'where':      ('user_id = ?', 'venue_id = ?')}
             cursor.execute(util.query(**qry), (1 if util.to_bool(promo_perm) else 0,
                                                staff_user_id, venue_id))
     return True
Exemplo n.º 2
0
 def set(self, cursor=None, facebook_id=None, user_id=None, venue_id=None,
         name=None, address=None, country=None, phone=None, email=None,
         email_verified=None, category_id=None, headline=None, tonight=None,
         website=None, facebook=None, twitter=None, v_facebook_id=None,
         twitter_id=None, twitter_token=None, twitter_secret=None, lat=None,
         lon=None, official=None, verified=None, customer_spend=None,
         authenticated=None, creator_version=None, **kwargs):
     data = {'name':           name,
             'address':        address,
             'country':        country,
             'phone':          phone,
             'email':          email,
             'email_verified': util.to_bool(email_verified),
             'category_id':    util.to_int(category_id),
             'headline':       headline,
             'tonight':        tonight,
             'website':        website,
             'facebook':       facebook,
             'twitter':        twitter,
             'facebook_id':    v_facebook_id,
             'twitter_id':     twitter_id,
             'twitter_token':  twitter_token,
             'twitter_secret': twitter_secret,
             'lat':            util.to_float(lat),
             'lon':            util.to_float(lon),
             'official':       util.to_bool(official),
             'verified':       util.to_bool(verified),
             'customer_spend': util.to_float(customer_spend),
             'authenticated':  util.to_bool(authenticated),
             'creator':        user_id,
             'creator_version': creator_version}
     columns = []
     values = []
     for key, val in data.iteritems():
         if val != None:
             columns.append(key)
             values.append(val)
     if venue_id:
         qry = {'update':     'venues',
                'set_values': columns,
                'where':      'id = ?'}
         values.append(venue_id)
         cursor.execute(util.query(**qry), values)
     else:
         qry = {'insert_into': 'venues',
                'columns':     columns}
         cursor.execute(util.query(**qry), values)
     cursor.execute(util.query(last_id=True))
     return int(cursor.fetchone().identity)
Exemplo n.º 3
0
 def __init__(self, logger, config):
     self.sock = []
     self.log = logger
     self.config = config
     self.uuidpool = {} # key MAC address value, value UUID value
     self.ippool = {} # key MAC address string, value assigned IP string
     self.filepool = {} # key IP string, value pathname
     self.states = {} # key MAC address string, value client state
     name_ = PRODUCT_NAME.split('-')
     name_[0] = 'bootp'
     self.netconfig = get_iface_config(self.config.get_bootp_bind_interface())
     if not self.netconfig:
         raise BootpError('Unable to detect network configuration')
     keys = sorted(self.netconfig.keys())
     self.log.info('Using %s' % ', '.join(map(':'.join,
                             zip(keys, [self.netconfig[k] for k in keys]))))
     access = self.config.get_bootp_acl_type()
     if not access:
         self.acl = None
     else:
         access = access.lower()
         if access not in self.ACCESS_LOCAL + self.ACCESS_REMOTE:
             raise BootpError('Invalid access mode: %s' % access)
         if not self.config.has_section(access):
             raise BootpError("Missing access section '%s'" % access)
         self.acl = {}
         if access in self.ACCESS_LOCAL:
             for entry in self.config.options(access):
                 self.acl[entry.upper()] = \
                     to_bool(self.config.get(access, entry))
     self.access = access
Exemplo n.º 4
0
 def __init__(self, logger, config):
     self.sock = []
     self.log = logger
     self.config = config
     self.uuidpool = {}  # key MAC address value, value UUID value
     self.ippool = {}  # key MAC address string, value assigned IP string
     self.filepool = {}  # key IP string, value pathname
     self.states = {}  # key MAC address string, value client state
     name_ = PRODUCT_NAME.split('-')
     name_[0] = 'bootp'
     self.netconfig = get_iface_config(
         self.config.get_bootp_bind_interface())
     if not self.netconfig:
         raise BootpError('Unable to detect network configuration')
     keys = sorted(self.netconfig.keys())
     self.log.info('Using %s' % ', '.join(
         map(':'.join, zip(keys, [self.netconfig[k] for k in keys]))))
     access = self.config.get_bootp_acl_type()
     if not access:
         self.acl = None
     else:
         access = access.lower()
         if access not in self.ACCESS_LOCAL + self.ACCESS_REMOTE:
             raise BootpError('Invalid access mode: %s' % access)
         if not self.config.has_section(access):
             raise BootpError("Missing access section '%s'" % access)
         self.acl = {}
         if access in self.ACCESS_LOCAL:
             for entry in self.config.options(access):
                 self.acl[entry.upper()] = \
                     to_bool(self.config.get(access, entry))
     self.access = access
Exemplo n.º 5
0
 def get(self, cursor=None, user_id=None, venue_id=None, getall=None,
         level=None, from_time=None, until_time=None, **kwargs):
     red = {'select': 'COUNT(id)',
            'table':  'promotion_redemptions',
            'where':  'promotion_id = promotions.id'}
     promo_qry = {'select':   ['id', 'title', 'description',
                               'passcode', 'start', '[end]', 'maximum',
                               'creator', 'level',
                               '(' + util.query(**red) + ') AS redemptions'],
                  'table':    'promotions',
                  'where':    ['venue_id = ?', 'hidden != 1'],
                  'order_by': 'id DESC'}
     if from_time and until_time:
         own_red = {'select': 'COUNT(id)',
                    'table':  'promotion_redemptions',
                    'where':  ('promotion_id = promotions.id', 'time >= ' + from_time, 'time < ' + until_time, 'user_id = ' + str(user_id))}
         promo_qry['select'].append('(' + util.query(**own_red) + ') AS own_redemptions')
     if not util.to_bool(getall):
         promo_qry['limit'] = 1
         promo_qry['where'].append(str(util.now()) + ' >= start')
         promo_qry['where'].append('([end] = 0 OR [end] > ' + str(util.now()) + ')')
         promo_qry['where'].append('(maximum = 0 OR (' + util.query(**red) + ') < maximum)')
         promo_qry['where'].append(level + ' >= level')
         promo_qry['order_by'] = 'level DESC, id DESC'
         cursor.execute(util.query(**promo_qry), (venue_id,))
         row = cursor.fetchone()
         if row:
             return {t[0]: val for t, val in zip(cursor.description, row)}
         else:
             return None
     cursor.execute(util.query(**promo_qry), (venue_id,))
     return [util.row_to_dict(cursor, row) for row in cursor.fetchall()]
Exemplo n.º 6
0
 def set(self, cursor=None, user_id=None, venue_id=None, delete=None,
         promotion_id=None, title=None, description=None, start=None,
         end=None, maximum=None, passcode=None, level=None, **kwargs):
     if util.to_bool(delete) and promotion_id:
         qry = {'update':     'promotions',
                'set_values': ('hidden'),
                'where':      'id = ?'}
         cursor.execute(util.query(**qry), (1, promotion_id))
     elif promotion_id:
         qry = {'update':     'promotions',
                'set_values': ('title', 'description', 'start', '[end]',
                               'maximum', 'passcode', 'venue_id', 'level'),
                'where':      'id = ?'}
         cursor.execute(util.query(**qry), (title, description, start, end,
                                            maximum, passcode, venue_id,
                                            level, promotion_id))
     else:
         qry = {'insert_into': 'promotions',
                'columns':      ('title', 'description', 'start', '[end]',
                                 'maximum', 'creator', 'passcode',
                                 'venue_id', 'level')}
         cursor.execute(util.query(**qry), (title, description, start, end,
                                            maximum, user_id, passcode,
                                            venue_id, level))
     return True
Exemplo n.º 7
0
    def __init__(self, username, password, latitude, longitude,
                 language='de', server_url=_SERVER_URL, skip_upload=False,
                 post_interval=300):
        """Initialize an instances of AWEKASThread.

        Required parameters:

          username: AWEKAS user name

          password: AWEKAS password

          language: Possible values include de, en, it, fr, nl
          Default is de

          latitude: Station latitude in decimal degrees
          Default is station latitude

          longitude: Station longitude in decimal degrees
          Default is station longitude
        
        Optional parameters:
        
          server_url: URL of the server
          Default is the AWEKAS site

          skip_upload: debugging option to display data but do not upload
          Default is False

          post_interval: The interval in seconds between posts. AWEKAS requests
          that uploads happen no more often than 5 minutes, so this should be
          set to no less than 300. Default is 300

          max_backlog: How many records are allowed to accumulate in the queue
          before the queue is trimmed. Default is sys.maxint (essentially,
          allow any number).

          log_success: If True, log a successful post in the system log.
          Default is True.
      
          log_failure: If True, log an unsuccessful post in the system log.
          Default is True.
          
          timeout: How long to wait for the server to respond before giving up.
          Default is 60 seconds

          max_tries: How many times to try the post before giving up.
          Default is 3

          retry_wait: How long to wait between retries when failures.
          Default is 5 seconds.
        """
        super(AWEKASThread, self).__init__(protocol_name='AWEKAS',
                                           post_interval=post_interval)
        self.username = username
        self.password = password
        self.latitude = float(latitude)
        self.longitude = float(longitude)
        self.language = language
        self.server_url = server_url
        self.skip_upload = to_bool(skip_upload)
Exemplo n.º 8
0
 def set(self, cursor=None, user_id=None, venue_id=None, following=None,
         **kwargs):
     qry = {'select':   'id',
            'table':    'venue_followers',
            'where':    ('user_id = ?', 'venue_id = ?'),
            'order_by': 'id',
            'limit':     1}
     cursor.execute(util.query(**qry), (user_id, venue_id))
     res = cursor.fetchone()
     if util.to_bool(following) and not res:
         qry = {'insert_into': 'venue_followers',
                'columns':      ('user_id', 'venue_id')}
         cursor.execute(util.query(**qry), (user_id, venue_id))
     elif not util.to_bool(following) and res:
         qry = {'delete': 'venue_followers',
                'where': ('user_id = ?', 'venue_id = ?')}
         cursor.execute(util.query(**qry), (user_id, venue_id))
     return True
Exemplo n.º 9
0
 def __init__(self, logger, parser):
     address = ('localhost', int(parser.get('httpd', 'port', '80')))
     HTTPServer.__init__(self, address, self.ReqHandler)
     self.log = logger
     self.uuids = {}
     access = 'uuid'
     if parser.has_section(access):
         for entry in parser.options(access):
             self.uuids[entry.upper().strip()] = \
                 to_bool(parser.get(access, entry))
Exemplo n.º 10
0
 def __init__(self, logger, parser):
     address = ('localhost', int(parser.get('httpd', 'port', '80')))
     HTTPServer.__init__(self, address, self.ReqHandler)
     self.log = logger
     self.uuids = {}
     access = 'uuid'
     if parser.has_section(access):
         for entry in parser.options(access):
             self.uuids[entry.upper().strip()] = \
                 to_bool(parser.get(access, entry))
Exemplo n.º 11
0
    def __post__(self, env, email, firstname, lastname, gender, language,
                 protected):
        self.app.update_user_details(self.username, email, firstname, lastname,
                                     gender, language, util.to_bool(protected))

        v = view.JSONView(200)
        m = self.app.get_full_user_details(self.username)
        v.bind(m)

        return v
Exemplo n.º 12
0
class BootpServer:
    """BOOTP Server
       Implements bootstrap protocol"""

    ACCESS_LOCAL = ['uuid', 'mac']  # Access modes, defined locally
    ACCESS_REMOTE = ['http']  # Access modes, remotely retrieved
    (ST_IDLE, ST_PXE, ST_DHCP) = range(3)  # Current state

    def __init__(self, logger, config):
        self.sock = []
        self.log = logger
        self.config = config
        self.uuidpool = {}  # key MAC address value, value UUID value
        self.ippool = {}  # key MAC address string, value assigned IP string
        self.filepool = {}  # key IP string, value pathname
        self.states = {}  # key MAC address string, value client state
        name_ = PRODUCT_NAME.split('-')
        name_[0] = 'bootp'
        self.bootp_section = '_'.join(name_)
        self.pool_start = self.config.get(self.bootp_section, 'pool_start')
        if not self.pool_start:
            raise BootpError('Missing pool_start definition')
        self.pool_count = int(
            self.config.get(self.bootp_section, 'pool_count', '10'))
        self.netconfig = get_iface_config(self.pool_start)
        if not self.netconfig:
            raise BootpError('Unable to detect network configuration')
        keys = sorted(self.netconfig.keys())
        self.log.info('Using %s' % ', '.join(
            map(':'.join, zip(keys, [self.netconfig[k] for k in keys]))))
        nlist = self.config.get(self.bootp_section, 'notify')
        self.notify = []
        if nlist:
            try:
                nlist = nlist.split(';')
                for n in nlist:
                    n = n.strip().split(':')
                    self.notify.append((n[0], int(n[1])))
            except Exception, e:
                raise BootpError('Invalid notification URL: %s' % str(e))
        access = self.config.get(self.bootp_section, 'access')
        if not access:
            self.acl = None
        else:
            access = access.lower()
            if access not in self.ACCESS_LOCAL + self.ACCESS_REMOTE:
                raise BootpError('Invalid access mode: %s' % access)
            if not self.config.has_section(access):
                raise BootpError("Missing access section '%s'" % access)
            self.acl = {}
            if access in self.ACCESS_LOCAL:
                for entry in self.config.options(access):
                    self.acl[entry.upper()] = \
                        to_bool(self.config.get(access, entry))
        self.access = access
Exemplo n.º 13
0
 def get(self, cursor=None, venue_id=None, from_time=None, until_time=None,
         own=None, user_id=None, **kwargs):
     qry = {'select':   'COUNT(id) AS cnt',
            'table':    'venue_rsvps',
            'where':    ('venue_id = ?', 'maybe = 1', 'going = 0',
                         'time >= ?', 'time < ?')}
     values = (venue_id, from_time, until_time)
     if util.to_bool(own):
         qry['where'] += ('user_id = ?',)
         values += (user_id,)
     cursor.execute(util.query(**qry), values)
     maybe = cursor.fetchone().cnt
     qry = {'select':   'COUNT(id) AS cnt',
            'table':    'venue_rsvps',
            'where':    ('venue_id = ?', 'going = 1',
                         'time >= ?', 'time < ?')}
     if util.to_bool(own):
         qry['where'] += ('user_id = ?',)
     cursor.execute(util.query(**qry), values)
     going = cursor.fetchone().cnt
     return {'maybe': maybe, 'going': going}
Exemplo n.º 14
0
 def _build(self, input_, core_state):
     action, env_output = input_
     env_outputs = environments.StepOutput(
         reward=env_output.reward[None, ...],
         info=nest.map_structure(lambda t: t[None, ...], env_output.info),
         done=to_bool(tf.cast(env_output.done, tf.int64)[None, ...]),
         observation=(tf.to_float(env_output.observation[0])[None, ...],
                      tf.zeros(())))
     actions = action[None, ...]
     outputs, core_state, custom_rnn_output, torso_outputs = self.unroll(
         actions, env_outputs, core_state)
     return nest.map_structure(lambda t: tf.squeeze(t, 0), outputs), core_state, \
            custom_rnn_output, torso_outputs
Exemplo n.º 15
0
 def set(self, cursor=None, user_id=None, venue_id=None, maybe=None,
         going=None, from_time=None, until_time=None, **kwargs):
     qry = {'select':   'id',
            'table':    'venue_rsvps',
            'where':    ('user_id = ?', 'venue_id = ?',
                         'time >= ?', 'time < ?'),
            'order_by': 'id',
            'limit':     1}
     cursor.execute(util.query(**qry), (user_id, venue_id, from_time,
                    until_time))
     res = cursor.fetchone()
     if res:
         values = []
         columns = []
         if maybe:
             values.append(util.to_bool(maybe))
             columns.append('maybe')
         if going:
             values.append(util.to_bool(going))
             columns.append('going')
         values.append(res.id)
         qry = {'update':     'venue_rsvps',
                'set_values': columns,
                'where':      'id = ?'}
         cursor.execute(util.query(**qry), values)
     else:
         values = [user_id, venue_id, util.now()]
         columns = ['user_id', 'venue_id', 'time']
         if maybe:
             values.append(util.to_bool(maybe))
             columns.append('maybe')
         if going:
             values.append(util.to_bool(going))
             columns.append('going')
         qry = {'insert_into': 'venue_rsvps',
                'columns':     columns}
         cursor.execute(util.query(**qry), values)
     return True
Exemplo n.º 16
0
    def get(self) -> Tuple[Dict, int]:  # type: ignore
        api10 = request.args.get("api10")
        api14 = request.args.get("api14")
        entity = request.args.get("entity")
        entity12 = request.args.get("entity12")
        identifier = request.args.get("id")
        status = request.args.get("status")
        related = request.args.get("related")

        kwargs: Dict[str, Any] = {}

        if status:
            kwargs["status"] = status

        if identifier:
            kwargs["_id"] = identifier

        if api10:
            kwargs["api10"] = api10

        if api14:
            kwargs["api14"] = api14

        if entity12:
            kwargs["entity12"] = entity12

        if entity:
            kwargs["entity"] = entity

        if related is not None:
            related = util.to_bool(related)

        if related and not entity12:
            objs = self.model.objects(**kwargs).only("entity12").all()
            # kwargs = {"entity12__in": list({x.entity12 for x in objs})}
            entity12 = util.reduce(list({x.entity12 for x in objs}))

            if isinstance(entity12, (list, set)):
                key = "entity12__in"
            else:
                key = "entity12"

            if entity12:
                kwargs = {key: entity12}

        logger.debug(f"production get: {kwargs}")  # noqa

        return self._get(**kwargs), 200
Exemplo n.º 17
0
def _set(name, value):
    if name.startswith('_') or name not in globals():
        raise ValueError("Invalid configuration value '{}'".format(name))
    t = type(globals()[name])
    if t in (callable, property):
        raise ValueError('Cannot set derived configuration value')

    try:
        if t == bool:
            globals()[name] = util.to_bool(value)
        else:
            globals()[name] = t(value)
    except ValueError:
        raise ValueError("Invalid type for configuration value '{}', must be: {}".format(name, t))

    logging.debug("Configuration value {} = {}".format(name, value))
Exemplo n.º 18
0
 def set(self, cursor=None, user_id=None, venue_id=None, caption=None,
         hide=None, post_id=None, image=None, **kwargs):
     if post_id and util.to_bool(hide):
         qry = {'update':     'posts',
                'set_values': ('hidden'),
                'where':      'id = ?'}
         cursor.execute(util.query(**qry), ('1', post_id))
         return post_id
     else:
         qry = {'insert_into': 'posts',
                'columns':     ('user_id', 'venue_id', 'caption', 'time')}
         cursor.execute(util.query(**qry), (user_id, venue_id, caption,
                                            util.now()))
         cursor.execute(util.query(last_id=True))
         post_added = int(cursor.fetchone().identity)
         azureutil.store(image.file, 'post', str(post_added))
         return post_added
Exemplo n.º 19
0
def _set(name, value):
    if name.startswith('_') or name not in globals():
        raise ValueError("Invalid configuration value '{}'".format(name))
    t = type(globals()[name])
    if t in (callable, property):
        raise ValueError('Cannot set derived configuration value')

    try:
        if t == bool:
            globals()[name] = util.to_bool(value)
        else:
            globals()[name] = t(value)
    except ValueError:
        raise ValueError(
            "Invalid type for configuration value '{}', must be: {}".format(
                name, t))

    logging.debug("Configuration value {} = {}".format(name, value))
Exemplo n.º 20
0
 def retrieve(self, cursor=None, user_id=None, term=None):
     if util.to_bool(term):
         qry = {'select':    ['id',
                              'facebook_id',
                              'forename',
                              'surname'
                              ],
                'table':     'users',
                'where':     ("CONCAT(forename, \' \', surname) LIKE ?",),
                'order_by':  'surname ASC, forename ASC'}
         cursor.execute(util.query(**qry), ("%" + term.replace(' ', "%") + "%",))
         return [util.row_to_dict(cursor, row) for row in cursor]
     else:
         qry = {'select':    ['id',
                              'facebook',
                              'twitter',
                              'forename',
                              'surname',
                              'age',
                              'birth_day',
                              'birth_month',
                              'birth_year',
                              'gender',
                              'employee',
                              'joined',
                              'country',
                              'language',
                              'email',
                              'top5',
                              'save_locally',
                              'last_login',
                              'last_facebook',
                              'last_twitter'
                              ],
                'table':     'users',
                'order_by':  'id'}
         qry['select'].append('twitter_id')
         qry['select'].append('twitter_token')
         qry['select'].append('twitter_secret')
         qry.update({'where': 'id = ?', 'limit': 1})
         cursor.execute(util.query(**qry), (user_id,))
         res = cursor.fetchone()
         return util.row_to_dict(cursor, res)
Exemplo n.º 21
0
	def update_user_details(self, username, email, firstname, lastname, gender, language, protected):
		# validate parameters:
		if not validate_email(email):
			raise exception.InvalidParameterException("email")

		if not validate_firstname(firstname):
			raise exception.InvalidParameterException("firstname")

		if not validate_lastname(lastname):
			raise exception.InvalidParameterException("lastname")

		if not validate_gender(gender):
			raise exception.InvalidParameterException("gender")

		if not validate_language(language):
			raise exception.InvalidParameterException("language")

		if not isinstance(protected, bool):
			if isinstance(protected, str) and (protected.lower() in ["true", "false"]):
				protected = util.to_bool(protected)
			else:
				raise exception.InvalidParameterException("protected")

		# update user details:
		with self.__create_db_connection__() as conn:
			with conn.enter_scope() as scope:
				self.__test_active_user__(scope, username)

				# test email
				if not self.__user_db.user_can_change_email(scope, username, email):
					raise exception.ConflictException("Email address is already assigned.")

				# update user details:
				self.__user_db.update_user_details(scope, username, email, firstname, lastname, gender, language, protected)

				scope.complete()
Exemplo n.º 22
0
    def handle(self, sock, addr, data):
        sender = addr
        self.log.info('Sender: %s on socket %s' % (addr, sock.getsockname()))
        if len(data) < DHCPFormatSize:
            self.log.error('Cannot be a DHCP or BOOTP request - too small!')
        tail = data[DHCPFormatSize:]
        buf = list(struct.unpack(DHCPFormat, data[:DHCPFormatSize]))
        if buf[BOOTP_OP] != BOOTREQUEST:
            self.log.warn('Not a BOOTREQUEST')
            return
        options = self.parse_options(tail)
        if options is None:
            self.log.warn('Error in option parsing, ignore request')
            return

        # Extras (DHCP options)
        try:
            dhcp_msg_type = ord(options[53][0])
        except KeyError:
            dhcp_msg_type = None

        server_addr = self.netconfig['address']
        mac_addr = buf[BOOTP_CHADDR][:6]
        gi_addr = buf[BOOTP_GIADDR][:4]
        mac_str = ':'.join(['%02X' % ord(x) for x in mac_addr])
        gi_str = '.'.join(['%d' % ord(x) for x in gi_addr])
        self.log.debug("Gateway address: %s" % gi_str)
        # is the UUID received (PXE mode)
        if 97 in options and len(options[97]) == 17:
            uuid = options[97][1:]
            pxe = True
            self.log.info('PXE UUID has been received')
        # or retrieved from the cache (DHCP mode)
        else:
            uuid = self.uuidpool.get(mac_addr, None)
            pxe = False
            self.log.info('PXE UUID not present in request')
        uuid_str = uuid and ('%s-%s-%s-%s-%s' % \
            tuple([hexlify(x) for x in uuid[0:4], uuid[4:6], uuid[6:8],
                                       uuid[8:10], uuid[10:16]])).upper()
        if uuid_str:
            self.log.info('UUID is %s for MAC %s' % (uuid_str, mac_str))

        hostname = ''
        filename = ''

        # Basic state machine
        currentstate = self.states.setdefault(mac_str, self.ST_IDLE)
        newstate = currentstate
        if currentstate == self.ST_IDLE:
            if pxe and (dhcp_msg_type == DHCP_DISCOVER):
                # BIOS is booting up, and try to locate a DHCP server
                newstate = self.ST_PXE
        elif currentstate == self.ST_PXE:
            if not pxe and (dhcp_msg_type == DHCP_REQUEST):
                # OS is booting up, and confirm a previous DHCP dicovery
                newstate = self.ST_DHCP
        else:  # currentstate == self.ST_DHCP
            if pxe:
                # OS was running but the BIOS is performing a DHCP request:
                # board has been restarted
                newstate = self.ST_PXE

        # if the state has not evolved from idle, there is nothing to do
        if newstate == self.ST_IDLE:
            self.log.info('Request from %s ignored (idle state)' % mac_str)
            sdhcp = self.config.get_bootp_allow_simple_dhcp()
            simple_dhcp = sdhcp and to_bool(sdhcp)
            if not simple_dhcp:
                return

        # construct reply
        buf[BOOTP_OP] = BOOTREPLY
        self.log.info('Client IP: %s' % socket.inet_ntoa(buf[7]))
        if buf[BOOTP_CIADDR] == '\x00\x00\x00\x00':
            self.log.debug('Client needs its address')
            host_data = self.get_host_data_for_mac(mac_str)
            ipaddr = host_data['address']

            self.log.debug("IPADDR: {0}".format(ipaddr))
            if not ipaddr:
                self.log.error("Can't get IP address!")
                return
            ip = None
            if mac_str in self.ippool:
                ip = self.ippool[mac_str]
                self.log.info('Lease for MAC %s already defined as IP %s' % \
                                (mac_str, ip))
            else:
                self.ippool[mac_str] = ipaddr
                ip = ipaddr

            if not ip:
                #raise BootpError('No more IP available in definined pool')
                self.log.error("Can not find IP assigned to mac: %s" % mac_str)
                return

            mask = iptoint(self.netconfig['mask'])
            reply_broadcast = iptoint(ip) & mask
            reply_broadcast |= (~mask) & ((1 << 32) - 1)
            buf[BOOTP_YIADDR] = socket.inet_aton(ip)
            buf[BOOTP_SECS] = 0
            buf[BOOTP_FLAGS] = BOOTP_FLAGS_NONE
            addr = (inttoip(reply_broadcast), addr[1])
            self.log.debug('Reply to: %s:%s' % addr)
        else:
            buf[BOOTP_YIADDR] = buf[BOOTP_CIADDR]
            ip = socket.inet_ntoa(buf[BOOTP_YIADDR])
        buf[BOOTP_SIADDR] = socket.inet_aton(server_addr)
        if gi_addr:
            self.log.debug('Reply via gateway: %s' % gi_str)
            buf[BOOTP_GIADDR] = socket.inet_aton(gi_str)
        # sname
        buf[BOOTP_SNAME] = \
            '.'.join([host_data['hostname'], host_data['domain']])
        # file
        buf[BOOTP_FILE] = 'boot_file' in host_data and host_data[
            'boot_file'] or self.config.get_bootp_default_boot_file()

        if not dhcp_msg_type:
            self.log.warn('No DHCP message type found, discarding request')
            return
        if dhcp_msg_type == DHCP_DISCOVER:
            self.log.debug('DHCP DISCOVER')
            dhcp_reply = DHCP_OFFER
            self.log.info('Offering lease for MAC %s: IP %s' % \
                          (mac_str, ip))
        elif dhcp_msg_type == DHCP_REQUEST:
            self.log.debug('DHCP REQUEST')
            dhcp_reply = DHCP_ACK
            self.log.info('New lease for MAC %s: IP %s' % \
                          (mac_str, ip))
        elif dhcp_msg_type == DHCP_RELEASE:
            self.log.info('DHCP RELEASE')
            if not self.notify:
                return
        elif dhcp_msg_type == DHCP_INFORM:
            self.log.info('DHCP INFORM')
            return
        elif dhcp_msg_type == DHCP_DECLINE:
            self.log.debug('DHCP DECLINE')
            return
        else:
            self.log.error('Unmanaged DHCP message: %d' % dhcp_msg_type)
            return

        pkt = struct.pack(DHCPFormat, *buf)
        pkt += struct.pack('!BBB', DHCP_MSG, 1, dhcp_reply)
        #server = socket.inet_aton(server_addr)
        # FIXME: Hardcoded relay and netmask
        # Add something in lines of:
        # networks:
        #     10.40.13.160:
        #         netmask: 255.255.255.224
        #         gateway: 10.40.13.160
        #         dns: 10.40.1.80, 10.40.1.81
        server = socket.inet_aton('10.40.13.161')
        pkt += struct.pack('!BB4s', DHCP_SERVER, 4, server)
        #mask = socket.inet_aton(self.netconfig['mask'])
        mask = socket.inet_aton('255.255.255.224')
        pkt += struct.pack('!BB4s', DHCP_IP_MASK, 4, mask)
        pkt += struct.pack('!BB4s', DHCP_IP_GATEWAY, 4, server)
        # FIXME: Serving only default DNS for now
        dns = self.config.get_bootp_default_dns()

        if dns:
            if dns.lower() == 'auto':
                dns = self.get_dns_server() or socket.inet_ntoa(server)
            dns = socket.inet_aton(dns)
            pkt += struct.pack('!BB4s', DHCP_IP_DNS, 4, dns)
        pkt += struct.pack('!BBI', DHCP_LEASE_TIME, 4,
                           int(self.config.get_bootp_default_lease_time()))
        pkt += struct.pack('!BB', DHCP_END, 0)

        # do not attempt to produce a PXE-augmented response for
        # regular DHCP requests
        if pxe:
            extra_buf = self.build_pxe_options(options, server)
            if not extra_buf:
                return
        else:
            extra_buf = self.build_dhcp_options(hostname)

        # update the UUID cache
        if pxe:
            self.uuidpool[mac_addr] = uuid

        # send the response
        if gi_addr:
            sock.sendto(pkt + extra_buf, (gi_str, 67))
        else:
            sock.sendto(pkt + extra_buf, addr)

        # update the current state
        if currentstate != newstate:
            self.log.info('Moving from state %d to state %d' % \
                            (currentstate, newstate))
            self.states[mac_str] = newstate
Exemplo n.º 23
0
            self.log.info("Filename for IP %s is '%s'" % (ip, filename))
            self.filepool[ip] = filename
        else:
            self.log.debug('No filename defined for IP %s' % ip)

        pkt = struct.pack(DHCPFormat, *buf)
        pkt += struct.pack('!BBB', DHCP_MSG, 1, dhcp_reply)
        server = socket.inet_aton(server_addr)
        pkt += struct.pack('!BB4s', DHCP_SERVER, 4, server)

        mask = socket.inet_aton(self.config.get(
            self.bootp_section, 'netmask', self.netconfig['mask']))

        pkt += struct.pack('!BB4s', DHCP_IP_MASK, 4, mask)

        if to_bool(self.config.get(self.bootp_section, 'set_gateway', True)):
            gateway = socket.inet_aton(
                self.config.get(self.bootp_section, 'gateway', server_addr))
            pkt += struct.pack('!BB4s', DHCP_IP_GATEWAY, 4, gateway)

        dns = self.config.get(self.bootp_section,
                              'dns', None)
        if dns:
            if dns.lower() == 'auto':
                dns = self.get_dns_server() or socket.inet_ntoa(server)
            dns = socket.inet_aton(dns)
            pkt += struct.pack('!BB4s', DHCP_IP_DNS, 4, dns)
        pkt += struct.pack('!BBI', DHCP_LEASE_TIME, 4,
                           int(self.config.get(self.bootp_section, 'lease_time',
                                               str(24*3600))))
        pkt += struct.pack('!BB', DHCP_END, 0)
Exemplo n.º 24
0
from db import db
from main import app

from tests.models import TestModel  # noqa
from tests.utils import MockAsyncDispatch
import util
from util.jsontools import load_json
import calc  # noqa

ECHO = False

import pytest

# * custom markers
pytest.mark.cionly = pytest.mark.skipif(
    not util.to_bool(os.getenv("CI")),
    reason="run on CI only",
)


@pytest.fixture(scope="session")
def celery_config():
    return config.CeleryConfig.items()


@pytest.fixture(scope="session")
def celery_worker_parameters():
    return config.CeleryConfig.items()


@pytest.fixture(scope="session")
Exemplo n.º 25
0
 def comment_exists(self, scope, id):
     return util.to_bool(
         execute_scalar(scope.get_handle(),
                        "select count(id) from object_comment where id=%s",
                        id))
Exemplo n.º 26
0
class BaseConfig:
    """Base configuration"""

    load_dotenv(".env")

    """ General """
    FLASK_ENV = os.getenv("FLASK_ENV", "development")
    DEFAULT_COLLECTION_INTERVAL = {"hours": 1}
    ENV_NAME = os.getenv("ENV_NAME", socket.gethostname())
    DEFAULT_PROJECTION = "wgs84"

    """ Sentry """
    SENTRY_ENABLED = to_bool(os.getenv("SENTRY_ENABLED"))
    SENTRY_DSN = os.getenv("SENTRY_DSN", None)
    SENTRY_LEVEL = os.getenv("SENTRY_LEVEL", 40)
    SENTRY_EVENT_LEVEL = os.getenv("SENTRY_EVENT_LEVEL", 40)
    SENTRY_ENV_NAME = os.getenv("SENTRY_ENV_NAME", ENV_NAME)
    SENTRY_RELEASE = f"{project}-{version}"

    """ Datadog """
    DATADOG_ENABLED = to_bool(os.getenv("DATADOG_ENABLED"))
    DATADOG_API_KEY = os.getenv("DATADOG_API_KEY", os.getenv("DD_API_KEY", None))
    DATADOG_APP_KEY = os.getenv("DATADOG_APP_KEY", os.getenv("DD_APP_KEY", None))
    DATADOG_DEFAULT_TAGS = {
        "environment": ENVIRONMENT_MAP.get(FLASK_ENV, FLASK_ENV),
        "service_name": project,
        "service_version": version,
    }

    """ Config """
    CONFIG_BASEPATH = "./config"
    COLLECTOR_CONFIG_PATH = abs_path(*os.getenv('COLLECTOR_CONFIG_PATH', CONFIG_BASEPATH+"/collector.yaml").rsplit("/", 1))  # type: ignore # noqa

    COLLECTOR_CONFIG_PATH = abs_path(CONFIG_BASEPATH, "collector.yaml")
    COLLECTOR_CONFIG = load_config(COLLECTOR_CONFIG_PATH)
    PARSER_CONFIG_PATH = abs_path(CONFIG_BASEPATH, "parsers.yaml")
    PARSER_CONFIG = load_config(PARSER_CONFIG_PATH)
    QUERY_PATH = abs_path(CONFIG_BASEPATH, "templates/queries")
    EXPORT_PATH = abs_path(CONFIG_BASEPATH, "templates/exports")

    """ Logging """
    LOG_LEVEL = os.getenv("LOG_LEVEL", 20)
    LOG_FORMAT = os.getenv("LOG_FORMAT", "funcname")
    WEB_LOG_REQUESTS = to_bool(os.getenv("WEB_LOG_REQUESTS", True))
    WEB_LOG_RESPONSES = to_bool(os.getenv("WEB_LOG_RESPONSES", True))
    WEB_LOG_SAMPLE_FRAC = float(os.getenv("WEB_LOG_SAMPLE_FRAC", 0.5))
    WEB_LOG_SLOW_RESPONSE_THRESHOLD = float(os.getenv("WEB_LOG_SLOW_RESPONSE_THRESHOLD", 3))  # seconds # noqa

    CELERY_LOG_LEVEL = os.getenv("CELERY_LOG_LEVEL", LOG_LEVEL)
    CELERY_LOG_FORMAT = os.getenv("CELERY_LOG_FORMAT", LOG_FORMAT)

    """ --------------- Database --------------- """

    DATABASE_DRIVER = os.getenv("DATABASE_DRIVER", "mongodb")
    DATABASE_USERNAME = os.getenv("DATABASE_USERNAME", None)
    DATABASE_PASSWORD = os.getenv("DATABASE_PASSWORD", None)
    DATABASE_HOST = os.getenv("DATABASE_HOST", "localhost")
    DATABASE_PORT = os.getenv("DATABASE_PORT", 27017)
    DATABASE_NAME = os.getenv("DATABASE_NAME", "default")
    DATABASE_AUTHENTICATION_SOURCE = "admin"
    DATABASE_URI = os.getenv("DATABASE_URI", None)
    REPLICA_SET = os.getenv("REPLICA_SET", None)
    # DATABASE_UUID_REPRESENTATION = "standard"
    # DATABASE_CONNECT = os.getenv("DATABASE_CONNECT", False)

    """ Celery """
    BROKER_URL = os.getenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
    CELERY_TASK_LIST = ["celery_queue.tasks"]
    CELERYD_TASK_TIME_LIMIT = to_int(os.getenv(
        "CELERYD_TASK_TIME_LIMIT", 60 * 60 * 12
    ))  # 12 hours
    CELERY_TASK_SERIALIZER = "json"
    CELERY_ACCEPT_CONTENT = ["json"]
    # CELERYD_MAX_TASKS_PER_CHILD = to_int(os.getenv("CELERYD_MAX_TASKS_PER_CHILD", 1000))
    # CELERYD_MAX_MEMORY_PER_CHILD = to_int(os.getenv("CELERYD_MAX_MEMORY_PER_CHILD", 250000))  # 250MB # noqa
    CELERY_ENABLE_REMOTE_CONTROL = False  # required for sqs
    CELERY_SEND_EVENTS = False  # required for sqs
    CELERY_DEFAULT_QUEUE = f"{project}-default"  # sqs queue name
    CELERY_ROUTES = ("celery_queue.routers.hole_direction_router",)
    CELERY_TASK_CREATE_MISSING_QUEUES = to_bool(os.getenv(
        "CELERY_TASK_CREATE_MISSING_QUEUES", False
    ))
    CELERY_TASK_ALWAYS_EAGER = to_bool(os.getenv("CELERY_TASK_ALWAYS_EAGER", False))

    """ Celery Beat """
    CELERYBEAT_SCHEDULER = "redbeat.RedBeatScheduler"
    CELERYBEAT_LOAD_ENDPOINTS: bool = to_bool(os.getenv("CELERYBEAT_LOAD_ENDPOINTS", True))
    CELERYBEAT_MAX_LOOP_INTERVAL: int = to_int(os.getenv("CELERYBEAT_MAX_LOOP_INTERVAL", 30))
    REDBEAT_REDIS_URL = os.getenv("IHS_CRON_URL")
    REDBEAT_KEY_PREFIX = f"{project}:"
    REDBEAT_LOCK_TIMEOUT = CELERYBEAT_MAX_LOOP_INTERVAL * 5

    """ API """
    API_CLIENT_TYPE = os.getenv("IHS_CLIENT_TYPE", "legacy")
    API_BASE_URL = os.getenv("IHS_URL")
    API_USERNAME = os.getenv("IHS_USERNAME")
    API_PASSWORD = os.getenv("IHS_PASSWORD")
    API_APP_KEY = os.getenv("IHS_APP_KEY")
    API_SYNC_WINDOW_MINUTES = os.getenv("IHS_SYNC_WINDOW_MINUTES", 1440 * 7)
    API_HEADERS = {
        "Username": API_USERNAME,
        "Password": API_PASSWORD,
        "Application": API_APP_KEY,
    }
    API_WSDL_DIR = abs_path(CONFIG_BASEPATH, "wsdl")
    API_WSDLS = {
        "session": abs_path(API_WSDL_DIR, "{version}/Session.wsdl"),
        "querybuilder": abs_path(API_WSDL_DIR, "{version}/QueryBuilder.wsdl"),
        "exportbuilder": abs_path(API_WSDL_DIR, "{version}/ExportBuilder.wsdl"),
    }
    API_DOMAIN = "US"
    TASK_BATCH_SIZE = to_int(os.getenv("IHS_TASK_BATCH_SIZE", 50))
    SIMULATE_EXPENSIVE_TASKS = to_bool(os.getenv("IHS_SIMULATE_EXPENSIVE_TASKS", False))

    @property
    def show(self):
        return [x for x in dir(self) if not x.startswith("_")]

    @property
    def api_params(self):
        return {
            key.lower().replace("api_", ""): getattr(self, key)
            for key in dir(self)
            if key.startswith("API_")
        }

    @property
    def datadog_params(self):
        return {
            key.lower().replace("datadog_", ""): getattr(self, key)
            for key in dir(self)
            if key.startswith("DATADOG_")
        }

    @property
    def sentry_params(self):
        return {
            key.lower().replace("sentry_", ""): getattr(self, key)
            for key in dir(self)
            if key.startswith("SENTRY_")
        }

    @property
    def endpoints(self):
        return self.COLLECTOR_CONFIG.endpoints

    @property
    def functions(self):
        return self.COLLECTOR_CONFIG.functions

    @property
    def database_params(self):
        return {
            key.lower().replace("database_", ""): getattr(self, key)
            for key in dir(self)
            if key.startswith("DATABASE_")
        }

    def database_uri(self, hide_password=False, include_auth_source=True):
        db = self.database_params
        username = db.get("username")
        password = "******" if hide_password else db.get("password")
        auth_source = (
            f"?authSource={db.get('authentication_source')}"
            if include_auth_source
            else ""
        )
        if self.REPLICA_SET:
            replica_set = (
                f"?replicaSet={self.REPLICA_SET}"
                if not auth_source
                else f"&replicaSet={self.REPLICA_SET}"
            )
        else:
            replica_set = ""

        driver = db.get("driver", "")
        host = db.get("host", "")
        port = db.get("port", "")
        dbname = db.get("name", "")
        at = "@" if username else ""
        colon = ":" if username is not None else ""
        username = username or ""
        password = password or ""
        return f"{driver}://{username}{colon}{password}{at}{host}:{port}/{dbname}{auth_source}{replica_set}"  # noqa

    def __repr__(self):
        """ Print configuration summary """
        hr = "-" * shutil.get_terminal_size().columns + "\n"
        tpl = "{name:>25} {value:<50}\n"
        string = ""
        string += tpl.format(name="app config:", value=APP_SETTINGS)
        string += tpl.format(name="flask app:", value=FLASK_APP)
        string += tpl.format(name="flask env:", value=self.FLASK_ENV)
        string += tpl.format(
            name="backend:",
            value=self.database_uri(hide_password=True, include_auth_source=False),
        )
        string += tpl.format(name="broker:", value=self.BROKER_URL)
        # string += tpl.format(name="result broker:", value=self.CELERY_RESULT_BACKEND)
        string += tpl.format(name="collector:", value=self.API_BASE_URL)
        return hr + string + hr
Exemplo n.º 27
0
	def object_exists(self, scope, guid):
		return util.to_bool(execute_scalar(scope.get_handle(), "select count(guid) from Object where guid=%s and deleted=false", guid))
Exemplo n.º 28
0
	def is_locked(self, scope, guid):
		return util.to_bool(execute_scalar(scope.get_handle(), "select locked from Object where guid=%s", guid))
Exemplo n.º 29
0
	def __post__(self, env, guid, up):
		self.app.vote(self.username, guid, util.to_bool(up))

		return self.__get_voting__(guid)
Exemplo n.º 30
0
 def get(self, cursor=None, user_id=None, term=None, following_only=None,
         my_lat=None, my_lon=None, distance=None, own=None, quiet=None,
         trending=None, from_time=None, until_time=None, promotions=None,
         level=None, around_me=None, **kwargs):
     subqry = {'select':   'COUNT(id)',
               'table':    'venue_followers',
               'where':    ('user_id = ' + str(user_id),
                            'venue_id = venues.id')}
     red = {'select': 'COUNT(id)',
            'table':  'promotion_redemptions',
            'where':  'promotion_id = promotions.id'}
     promoqry = {'select':   'COUNT(id)',
                 'table':    'promotions',
                 'where':    ('venue_id = venues.id',
                              str(util.now()) + ' >= start',
                              '([end] = 0 OR [end] > ' + str(util.now()) + ')',
                              '(maximum = 0 OR (' + util.query(**red) + ') < maximum)',
                              level + ' >= level',
                              'hidden != 1')}
     managerqry = {'select':   'COUNT(id)',
                   'table':    'venue_managers',
                   'where':    ('user_id = ' + str(user_id),
                               'venue_id = venues.id')}
     staffqry =  {'select':   'COUNT(id)',
                  'table':    'venue_staff',
                  'where':    ('user_id = ' + str(user_id),
                               'venue_id = venues.id')}
     staffppqry =  {'select':   'SUM(promo_perm)',
                    'table':    'venue_staff',
                    'where':    ('user_id = ' + str(user_id),
                                 'venue_id = venues.id')}
     fields = ['id', 'name', 'address', 'country', 'phone', 'email',
               'email_verified', 'category_id', 'headline', 'tonight',
               'website', 'facebook', 'twitter', 'facebook_id',
               'twitter_id', 'twitter_token', 'twitter_secret', 'lat',
               'lon', 'official', 'verified', 'customer_spend',
               'authenticated', 'creator',
               '(' + util.query(**managerqry) + ') AS manager',
               '(' + util.query(**staffqry) + ') AS staff',
               '(' + util.query(**staffppqry) + ') AS promo_perm',
               "(" + util.query(**subqry) + ") AS following",
               '(' + util.query(**promoqry) + ') AS promotions']
     order_by = ('name ASC',)
     if term:
         where = ("name LIKE ?",)
     elif util.to_bool(following_only):
         where = ("(" + util.query(**subqry) + ") > 0")
     elif own:
         where = ('(' + util.query(**managerqry) + ') = 1 OR (' + util.query(**staffqry) + ') = 1')
     elif my_lat and my_lon and distance:
         maybe = {'select':   'COUNT(id)',
                  'table':    'venue_rsvps',
                  'where':    ('maybe = 1', 'venue_id = venues.id',
                               'going = 0', 'time >= ?', 'time < ?')}
         going = {'select':   'COUNT(id)',
                  'table':    'venue_rsvps',
                  'where':    ('going = 1', 'venue_id = venues.id',
                               'time >= ?', 'time < ?')}
         if util.to_bool(quiet):
             order_by = ('(' + util.query(**maybe) +') + (' + util.query(**going) +') * 2 ASC',)
         elif util.to_bool(trending):
             order_by = ('(' + util.query(**maybe) +') + (' + util.query(**going) +') * 2 DESC',)
         else:
             order_by = ('((lat - ?) * (lat - ?) + (lon - ?) * (lon - ?)) ASC',)
         where = ('((lat - ?) * (lat - ?) + (lon - ?) * (lon - ?)) <= ? * ?',)
         if util.to_bool(promotions):
             where += ('(' + util.query(**promoqry) + ') > 0',)
         elif util.to_bool(quiet) or util.to_bool(trending):
             fields[0] = 'TOP(12) id'
         elif util.to_bool(around_me):
             psubqry = {'select':   'COUNT(id)',
                        'table':    'post_reports',
                        'where':    ('post_id = posts.id')}
             post_count = {'select':   'CASE WHEN COUNT(id) > 0 THEN 1 ELSE 0 END',
                           'table':    'posts',
                           'where':    ('posts.venue_id = venues.id',
                                        'hidden = 0',
                                        '(' + util.query(**psubqry) + ') < 3',
                                        'time > ' + str(util.now() - 691200))}
             order_by = ('(' + util.query(**post_count) + ') DESC',) + order_by
     else:
         where = ''
     qry = {'select':   fields,
            'table':    'venues',
            'where':    where,
            'order_by': order_by}
     if term:
         cursor.execute(util.query(**qry), ("%" + term + "%",))
         return [util.row_to_dict(cursor, row) for row in cursor]
     else:
         values = tuple()
         if my_lat and my_lon and distance:
             values += (float(my_lat), float(my_lat), float(my_lon),
                        float(my_lon), float(distance), float(distance))
             if util.to_bool(quiet) is None and util.to_bool(trending) is None:
                 values += (float(my_lat), float(my_lat), float(my_lon),
                            float(my_lon))
             else:
                 values += (from_time, until_time, from_time, until_time)
         cursor.execute(util.query(**qry), values)
         return [util.row_to_dict(cursor, row) for row in cursor]
Exemplo n.º 31
0
    def handle(self, sock, addr, data):
        sender = addr
        self.log.info('Sender: %s on socket %s' % (addr, sock.getsockname()))
        if len(data) < DHCPFormatSize:
            self.log.error('Cannot be a DHCP or BOOTP request - too small!')
        tail = data[DHCPFormatSize:]
        buf = list(struct.unpack(DHCPFormat, data[:DHCPFormatSize]))
        if buf[BOOTP_OP] != BOOTREQUEST:
            self.log.warn('Not a BOOTREQUEST')
            return
        options = self.parse_options(tail)
        if options is None:
            self.log.warn('Error in option parsing, ignore request')
            return

        # Extras (DHCP options)
        try:
            dhcp_msg_type = ord(options[53][0])
        except KeyError:
            dhcp_msg_type = None

        server_addr = self.netconfig['address']
        mac_addr = buf[BOOTP_CHADDR][:6]
        gi_addr = buf[BOOTP_GIADDR][:4]
        mac_str = ':'.join(['%02X' % ord(x) for x in mac_addr])
        gi_str = '.'.join(['%d' % ord(x) for x in gi_addr])
        self.log.debug("Gateway address: %s" % gi_str)
        # is the UUID received (PXE mode)
        if 97 in options and len(options[97]) == 17:
            uuid = options[97][1:]
            pxe = True
            self.log.info('PXE UUID has been received')
        # or retrieved from the cache (DHCP mode)
        else:
            uuid = self.uuidpool.get(mac_addr, None)
            pxe = False
            self.log.info('PXE UUID not present in request')
        uuid_str = uuid and ('%s-%s-%s-%s-%s' % \
            tuple([hexlify(x) for x in uuid[0:4], uuid[4:6], uuid[6:8],
                                       uuid[8:10], uuid[10:16]])).upper()
        if uuid_str:
            self.log.info('UUID is %s for MAC %s' % (uuid_str, mac_str))

        hostname = ''
        filename = ''

        # Basic state machine
        currentstate = self.states.setdefault(mac_str, self.ST_IDLE)
        newstate = currentstate
        if currentstate == self.ST_IDLE:
            if pxe and (dhcp_msg_type == DHCP_DISCOVER):
                # BIOS is booting up, and try to locate a DHCP server
                newstate = self.ST_PXE
        elif currentstate == self.ST_PXE:
            if not pxe and (dhcp_msg_type == DHCP_REQUEST):
                # OS is booting up, and confirm a previous DHCP dicovery
                newstate = self.ST_DHCP
        else: # currentstate == self.ST_DHCP
            if pxe:
                # OS was running but the BIOS is performing a DHCP request:
                # board has been restarted
                newstate = self.ST_PXE

        # if the state has not evolved from idle, there is nothing to do
        if newstate == self.ST_IDLE:
            self.log.info('Request from %s ignored (idle state)' % mac_str)
            sdhcp = self.config.get_bootp_allow_simple_dhcp()
            simple_dhcp = sdhcp and to_bool(sdhcp)
            if not simple_dhcp:
               return

        # construct reply
        buf[BOOTP_OP] = BOOTREPLY
        self.log.info('Client IP: %s' % socket.inet_ntoa(buf[7]))
        if buf[BOOTP_CIADDR] == '\x00\x00\x00\x00':
            self.log.debug('Client needs its address')
            host_data = self.get_host_data_for_mac(mac_str)
            ipaddr = host_data['address']

            self.log.debug("IPADDR: {0}".format(ipaddr))
            if not ipaddr:
                self.log.error("Can't get IP address!")
                return
            ip = None
            if mac_str in self.ippool:
                ip = self.ippool[mac_str]
                self.log.info('Lease for MAC %s already defined as IP %s' % \
                                (mac_str, ip))
            else:
                self.ippool[mac_str] = ipaddr
                ip = ipaddr

            if not ip:
                #raise BootpError('No more IP available in definined pool')
                self.log.error("Can not find IP assigned to mac: %s" % mac_str)
                return

            mask = iptoint(self.netconfig['mask'])
            reply_broadcast = iptoint(ip) & mask
            reply_broadcast |= (~mask)&((1<<32)-1)
            buf[BOOTP_YIADDR] = socket.inet_aton(ip)
            buf[BOOTP_SECS] = 0
            buf[BOOTP_FLAGS] = BOOTP_FLAGS_NONE
            addr = (inttoip(reply_broadcast), addr[1])
            self.log.debug('Reply to: %s:%s' % addr)
        else:
            buf[BOOTP_YIADDR] = buf[BOOTP_CIADDR]
            ip = socket.inet_ntoa(buf[BOOTP_YIADDR])
        buf[BOOTP_SIADDR] = socket.inet_aton(server_addr)
        if gi_addr:
            self.log.debug('Reply via gateway: %s' % gi_str)
            buf[BOOTP_GIADDR] = socket.inet_aton(gi_str)
        # sname
        buf[BOOTP_SNAME] = \
            '.'.join([host_data['hostname'], host_data['domain']])
        # file
        buf[BOOTP_FILE] = 'boot_file' in host_data and host_data['boot_file'] or self.config.get_bootp_default_boot_file()

        if not dhcp_msg_type:
            self.log.warn('No DHCP message type found, discarding request')
            return
        if dhcp_msg_type == DHCP_DISCOVER:
            self.log.debug('DHCP DISCOVER')
            dhcp_reply = DHCP_OFFER
            self.log.info('Offering lease for MAC %s: IP %s' % \
                          (mac_str, ip))
        elif dhcp_msg_type == DHCP_REQUEST:
            self.log.debug('DHCP REQUEST')
            dhcp_reply = DHCP_ACK
            self.log.info('New lease for MAC %s: IP %s' % \
                          (mac_str, ip))
        elif dhcp_msg_type == DHCP_RELEASE:
            self.log.info('DHCP RELEASE')
            if not self.notify:
                return
        elif dhcp_msg_type == DHCP_INFORM:
            self.log.info('DHCP INFORM')
            return
        elif dhcp_msg_type == DHCP_DECLINE:
            self.log.debug('DHCP DECLINE')
            return
        else:
            self.log.error('Unmanaged DHCP message: %d' % dhcp_msg_type)
            return

        pkt = struct.pack(DHCPFormat, *buf)
        pkt += struct.pack('!BBB', DHCP_MSG, 1, dhcp_reply)
        #server = socket.inet_aton(server_addr)
        # FIXME: Hardcoded relay and netmask
        # Add something in lines of:
        # networks:
        #     10.40.13.160:
        #         netmask: 255.255.255.224
        #         gateway: 10.40.13.160
        #         dns: 10.40.1.80, 10.40.1.81
        server = socket.inet_aton('10.40.13.161')
        pkt += struct.pack('!BB4s', DHCP_SERVER, 4, server)
        #mask = socket.inet_aton(self.netconfig['mask'])
        mask = socket.inet_aton('255.255.255.224')
        pkt += struct.pack('!BB4s', DHCP_IP_MASK, 4, mask)
        pkt += struct.pack('!BB4s', DHCP_IP_GATEWAY, 4, server)
        # FIXME: Serving only default DNS for now
        dns = self.config.get_bootp_default_dns()

        if dns:
            if dns.lower() == 'auto':
                dns = self.get_dns_server() or socket.inet_ntoa(server)
            dns = socket.inet_aton(dns)
            pkt += struct.pack('!BB4s', DHCP_IP_DNS, 4, dns)
        pkt += struct.pack('!BBI', DHCP_LEASE_TIME, 4,
                           int(self.config.get_bootp_default_lease_time()))
        pkt += struct.pack('!BB', DHCP_END, 0)

        # do not attempt to produce a PXE-augmented response for
        # regular DHCP requests
        if pxe:
            extra_buf = self.build_pxe_options(options, server)
            if not extra_buf:
                return
        else:
            extra_buf = self.build_dhcp_options(hostname)

        # update the UUID cache
        if pxe:
            self.uuidpool[mac_addr] = uuid

        # send the response
        if gi_addr:
            sock.sendto(pkt + extra_buf, (gi_str, 67))
        else:
            sock.sendto(pkt + extra_buf, addr)

        # update the current state
        if currentstate != newstate:
            self.log.info('Moving from state %d to state %d' % \
                            (currentstate, newstate))
            self.states[mac_str] = newstate
Exemplo n.º 32
0
class BaseConfig:
    """Base configuration"""

    FLASK_ENV = os.getenv("FLASK_ENV", "development")
    ENV_NAME = os.getenv("ENV_NAME", socket.gethostname())
    """ Datadog """
    DATADOG_ENABLED = to_bool(os.getenv("DATADOG_ENABLED", False))
    DATADOG_API_KEY = os.getenv("DATADOG_API_KEY", None)
    DATADOG_APP_KEY = os.getenv("DATADOG_APP_KEY", None)
    """ General """
    CONFIG_BASEPATH = "./config"
    """ Collector """
    COLLECTOR_CONFIG_PATH = make_config_path(CONFIG_BASEPATH, "collector.yaml")
    COLLECTOR_CONFIG = load_config(COLLECTOR_CONFIG_PATH)
    COLLECTOR_BASE_URL = os.getenv("FRACFOCUS_BASE_URL",
                                   "http://fracfocusdata.org")
    COLLECTOR_URL_PATH = os.getenv("FRACFOCUS_URL_PATH",
                                   "/digitaldownload/FracFocusCSV.zip")
    COLLECTOR_DOWNLOAD_PATH = os.getenv("FRACFOCUS_DOWNLOAD_PATH",
                                        "/tmp/fracfocus")
    COLLECTOR_WRITE_SIZE = int(os.getenv("FRACFOCUS_WRITE_SIZE", "10000"))
    COLLECTOR_FILE_PREFIX = os.getenv("FRACFOCUS_FILE_PREFIX",
                                      "FracFocusRegistry")
    """ Parser """
    PARSER_CONFIG_PATH = abs_path(CONFIG_BASEPATH, "parsers.yaml")
    PARSER_CONFIG = load_config(PARSER_CONFIG_PATH)
    """ Logging """
    LOG_LEVEL = os.getenv("LOG_LEVEL", logging.INFO)
    LOG_FORMAT = os.getenv("LOG_FORMAT", "json")
    WEB_LOG_REQUESTS = to_bool(os.getenv("WEB_LOG_REQUESTS", True))
    WEB_LOG_RESPONSES = to_bool(os.getenv("WEB_LOG_RESPONSES", True))
    WEB_LOG_SAMPLE_FRAC = float(os.getenv("WEB_LOG_SAMPLE_FRAC", 0.5))
    WEB_LOG_SLOW_RESPONSE_THRESHOLD = float(
        os.getenv("WEB_LOG_SLOW_RESPONSE_THRESHOLD", 3))  # seconds # noqa
    """ --------------- Sqlalchemy --------------- """

    DATABASE_DIALECT = os.getenv("DATABASE_DIALECT", "postgres")
    DATABASE_DRIVER = os.getenv("DATABASE_DRIVER",
                                get_default_driver(DATABASE_DIALECT))
    DATABASE_USERNAME = os.getenv("DATABASE_USERNAME", "")
    DATABASE_PASSWORD = os.getenv("DATABASE_PASSWORD", "")
    DATABASE_HOST = os.getenv("DATABASE_HOST", "localhost")
    DATABASE_PORT = os.getenv("DATABASE_PORT",
                              get_default_port(DATABASE_DRIVER))
    DATABASE_SCHEMA = os.getenv("DATABASE_SCHEMA", "fracfocus")
    DATABASE_NAME = os.getenv("DATABASE_NAME", "fracfocus")
    DATABASE_URL_PARAMS = {
        "drivername": DATABASE_DRIVER,
        "username": DATABASE_USERNAME,
        "password": DATABASE_PASSWORD,
        "host": DATABASE_HOST,
        "port": DATABASE_PORT,
        "database": DATABASE_NAME,
    }
    SQLALCHEMY_DATABASE_URI = str(make_url(DATABASE_URL_PARAMS))
    DEFAULT_EXCLUSIONS = ["updated_at", "inserted_at"]

    @property
    def show(self):
        return [x for x in dir(self) if not x.startswith("_")]

    @property
    def collector_params(self):
        return self.get_params_by_prefix("collector")

    @property
    def datadog_params(self):
        return self.get_params_by_prefix("datadog")

    @property
    def endpoints(self):
        return self.COLLECTOR_CONFIG.endpoints

    def __repr__(self):
        """ Print noteworthy configuration items """
        hr = "-" * shutil.get_terminal_size().columns + "\n"
        tpl = "{name:>25} {value:<50}\n"
        string = ""
        string += tpl.format(name="app config:", value=APP_SETTINGS)
        string += tpl.format(name="flask app:", value=FLASK_APP)
        string += tpl.format(name="flask env:", value=self.FLASK_ENV)
        string += tpl.format(name="backend:",
                             value=make_url(
                                 self.DATABASE_URL_PARAMS).__repr__())
        string += tpl.format(name="collector:", value=self.COLLECTOR_BASE_URL)
        return hr + string + hr

    def get_params_by_prefix(self, kw: str):
        """ Return all parameters that begin with the given string.

            Example: kw = "collector"

                Returns:
                    {
                        "base_url": "example.com/api",
                        "path": "path/to/data",
                        "endpoints": {...}
                    }
        """
        if not kw.endswith("_"):
            kw = kw + "_"
        return {
            key.lower().replace(kw.lower(), ""): getattr(self, key)
            for key in dir(self) if key.startswith(kw.upper())
        }
Exemplo n.º 33
0
    def handle(self, sock, addr, data):
        self.log.info('Sender: %s on socket %s' % (addr, sock.getsockname()))
        if len(data) < DHCPFormatSize:
            self.log.error('Cannot be a DHCP or BOOTP request - too small!')
        tail = data[DHCPFormatSize:]
        buf = list(struct.unpack(DHCPFormat, data[:DHCPFormatSize]))
        if buf[BOOTP_OP] != BOOTREQUEST:
            self.log.warn('Not a BOOTREQUEST')
            return
        options = self.parse_options(tail)
        if options is None:
            self.log.warn('Error in option parsing, ignore request')
            return

        # Extras (DHCP options)
        try:
            dhcp_msg_type = ord(options[53][0])
        except KeyError:
            dhcp_msg_type = None

        server_addr = self.netconfig['server']
        mac_addr = buf[BOOTP_CHADDR][:6]
        mac_str = '-'.join(['%02X' % ord(x) for x in mac_addr])
        # is the UUID received (PXE mode)
        if 97 in options and len(options[97]) == 17:
            uuid = options[97][1:]
            pxe = True
            self.log.info('PXE UUID has been received')
        # or retrieved from the cache (DHCP mode)
        else:
            uuid = self.uuidpool.get(mac_addr, None)
            pxe = False
            self.log.info('PXE UUID not present in request')
        uuid_str = uuid and ('%s-%s-%s-%s-%s' % \
            tuple([hexlify(x) for x in uuid[0:4], uuid[4:6], uuid[6:8],
                                       uuid[8:10], uuid[10:16]])).upper()
        if uuid_str:
            self.log.info('UUID is %s for MAC %s' % (uuid_str, mac_str))

        hostname = ''
        filename = ''

        # Basic state machine
        currentstate = self.states.setdefault(mac_str, self.ST_IDLE)
        newstate = currentstate
        if currentstate == self.ST_IDLE:
            if pxe and (dhcp_msg_type == DHCP_DISCOVER):
                # BIOS is booting up, and try to locate a DHCP server
                newstate = self.ST_PXE
        elif currentstate == self.ST_PXE:
            if not pxe and (dhcp_msg_type == DHCP_REQUEST):
                # OS is booting up, and confirm a previous DHCP dicovery
                newstate = self.ST_DHCP
        else:  # currentstate == self.ST_DHCP
            if pxe:
                # OS was running but the BIOS is performing a DHCP request:
                # board has been restarted
                newstate = self.ST_PXE

        # if the state has not evolved from idle, there is nothing to do
        if newstate == self.ST_IDLE:
            self.log.info('Request from %s ignored (idle state)' % mac_str)
            sdhcp = 'allow_simple_dhcp'
            simple_dhcp = \
                self.config.has_option(self.bootp_section, sdhcp) and \
                to_bool(self.config.get(self.bootp_section, sdhcp))
            if not simple_dhcp:
                return

        # if access control is enable
        if self.access:
            # remote access is always validated on each request
            if self.access in self.ACCESS_REMOTE:
                # need to query a host to grant or reject access
                import urlparse
                import urllib
                netloc = self.config.get(self.access, 'location')
                path = self.config.get(self.access, pxe and 'pxe' or 'dhcp')
                timeout = int(self.config.get(self.access, 'timeout', '5'))
                parameters = {'mac': mac_str}
                if uuid:
                    parameters['uuid'] = uuid_str
                if not pxe and mac_str in self.ippool:
                    parameters['ip'] = self.ippool[mac_str]
                item = uuid_str or mac_str
                # only bother the authentication host when a state change is
                # required.
                if currentstate != newstate:
                    query = urllib.urlencode(parameters)
                    urlparts = (self.access, netloc, path, query, '')
                    #print "\n"
                    #print "urlpart is ", urlparts
                    #print "\n"
                    url = urlparse.urlunsplit(urlparts)
                    self.log.info('Requesting URL: %s' % url)
                    import urllib2
                    import httplib
                    try:

                        #up = urllib2.urlopen(url, timeout=timeout)
                        proxy_handler = urllib2.ProxyHandler({})
                        opener = urllib2.build_opener(proxy_handler)
                        urllib2.install_opener(opener)
                        up = urllib2.urlopen(url, timeout=timeout)

                        for l in up:
                            try:
                                # Look for extra definition within the reply
                                k, v = [x.strip() for x in l.split(':')]
                                k = k.lower()
                                if k == 'client':
                                    hostname = v
                                if k == 'file':
                                    filename = v
                            except ValueError:
                                pass
                    except urllib2.HTTPError, e:
                        self.log.error('HTTP Error: %s' % str(e))
                        self.states[mac_str] = self.ST_IDLE
                        return
                    except urllib2.URLError, e:
                        self.log.critical('Internal error: %s' % str(e))
                        self.states[mac_str] = self.ST_IDLE
                        return
                    except httplib.HTTPException, e:
                        self.log.error('Server error: %s' % type(e))
                        self.states[mac_str] = self.ST_IDLE
                        return
Exemplo n.º 34
0
    def __post__(self, env, guid, up):
        self.app.vote(self.username, guid, util.to_bool(up))

        return self.__get_voting__(guid)
Exemplo n.º 35
0
 def object_exists(self, scope, guid):
     return util.to_bool(
         execute_scalar(
             scope.get_handle(),
             "select count(guid) from Object where guid=%s and deleted=false",
             guid))
Exemplo n.º 36
0
    def handle(self, sock, addr, data):
        self.log.info('Sender: %s on socket %s' % (addr, sock.getsockname()))
        if len(data) < DHCPFormatSize:
            self.log.error('Cannot be a DHCP or BOOTP request - too small!')
        tail = data[DHCPFormatSize:]
        buf = list(struct.unpack(DHCPFormat, data[:DHCPFormatSize]))
        if buf[BOOTP_OP] != BOOTREQUEST:
            self.log.warn('Not a BOOTREQUEST')
            return
        options = self.parse_options(tail)
        if options is None:
            self.log.warn('Error in option parsing, ignore request')
            return

        # Extras (DHCP options)
        try:
            dhcp_msg_type = ord(options[53][0])
        except KeyError:
            dhcp_msg_type = None

        server_addr = self.netconfig['server']
        mac_addr = buf[BOOTP_CHADDR][:6]
        mac_str = '-'.join(['%02X' % ord(x) for x in mac_addr])
        # is the UUID received (PXE mode)
        if 97 in options and len(options[97]) == 17:
            uuid = options[97][1:]
            pxe = True
            self.log.info('PXE UUID has been received')
        # or retrieved from the cache (DHCP mode)
        else:
            uuid = self.uuidpool.get(mac_addr, None)
            pxe = False
            self.log.info('PXE UUID not present in request')
        uuid_str = uuid and ('%s-%s-%s-%s-%s' % \
            tuple([hexlify(x) for x in uuid[0:4], uuid[4:6], uuid[6:8],
                                       uuid[8:10], uuid[10:16]])).upper()
        if uuid_str:
            self.log.info('UUID is %s for MAC %s' % (uuid_str, mac_str))

        hostname = ''
        filename = ''

        # Basic state machine
        currentstate = self.states.setdefault(mac_str, self.ST_IDLE)
        newstate = currentstate
        if currentstate == self.ST_IDLE:
            if pxe and (dhcp_msg_type == DHCP_DISCOVER):
                # BIOS is booting up, and try to locate a DHCP server
                newstate = self.ST_PXE
        elif currentstate == self.ST_PXE:
            if not pxe and (dhcp_msg_type == DHCP_REQUEST):
                # OS is booting up, and confirm a previous DHCP dicovery
                newstate = self.ST_DHCP
        else: # currentstate == self.ST_DHCP
            if pxe:
                # OS was running but the BIOS is performing a DHCP request:
                # board has been restarted
                newstate = self.ST_PXE

        # if the state has not evolved from idle, there is nothing to do
        if newstate == self.ST_IDLE:
            self.log.info('Request from %s ignored (idle state)' % mac_str)
            sdhcp = 'allow_simple_dhcp'
            simple_dhcp = \
                self.config.has_option(self.bootp_section, sdhcp) and \
                to_bool(self.config.get(self.bootp_section, sdhcp))
            if not simple_dhcp:
               return

        # if access control is enable
        if self.access:
            # remote access is always validated on each request
            if self.access in self.ACCESS_REMOTE:
                # need to query a host to grant or reject access
                import urlparse
                import urllib
                netloc = self.config.get(self.access, 'location')
                path = self.config.get(self.access, pxe and 'pxe' or 'dhcp')
                timeout = int(self.config.get(self.access, 'timeout', '5'))
                parameters = {'mac' : mac_str}
                if uuid:
                    parameters['uuid'] = uuid_str
                if not pxe and mac_str in self.ippool:
                    parameters['ip'] = self.ippool[mac_str]
                item = uuid_str or mac_str
                # only bother the authentication host when a state change is
                # required.
                if currentstate != newstate:
                    query = urllib.urlencode(parameters)
                    urlparts = (self.access, netloc, path, query, '')
                    #print "\n"
                    #print "urlpart is ", urlparts
                    #print "\n"
                    url = urlparse.urlunsplit(urlparts)
                    self.log.info('Requesting URL: %s' % url)
                    import urllib2
                    import httplib
                    try:
                        
                        #up = urllib2.urlopen(url, timeout=timeout)
                        proxy_handler = urllib2.ProxyHandler({})
                        opener = urllib2.build_opener(proxy_handler)
                        urllib2.install_opener(opener)
                        up = urllib2.urlopen(url, timeout=timeout)

                        for l in up:
                            try:
                                # Look for extra definition within the reply
                                k, v = [x.strip() for x in l.split(':')]
                                k = k.lower()
                                if k == 'client':
                                    hostname = v
                                if k == 'file':
                                    filename = v
                            except ValueError:
                                pass
                    except urllib2.HTTPError, e:
                        self.log.error('HTTP Error: %s' % str(e))
                        self.states[mac_str] = self.ST_IDLE
                        return
                    except urllib2.URLError, e:
                        self.log.critical('Internal error: %s' % str(e))
                        self.states[mac_str] = self.ST_IDLE
                        return
                    except httplib.HTTPException, e:
                        self.log.error('Server error: %s' % type(e))
                        self.states[mac_str] = self.ST_IDLE
                        return
Exemplo n.º 37
0
	def comment_exists(self, scope, id):
		return util.to_bool(execute_scalar(scope.get_handle(), "select count(id) from object_comment where id=%s", id))
Exemplo n.º 38
0
def build_learner(agent,
                  agent_state,
                  env_outputs,
                  agent_outputs,
                  learning_rate,
                  gradients_buffer=None,
                  episode_finished_mask=None,
                  torso_variables=None,
                  recurrent_variables=None):
    analysis_tensors = dict()

    learner_outputs, final_agent_state, custom_rnn_output, torso_outputs = agent.unroll(
        agent_outputs.action,
        env_outputs,
        agent_state,
        write_to_collection=True)
    rnn_v = custom_rnn_output[1][..., 0]
    rnn_thr = FLAGS.thr + agent.core.beta * custom_rnn_output[1][..., 1]
    rnn_pos = tf.nn.relu(rnn_v - rnn_thr)
    rnn_neg = tf.nn.relu(-rnn_v - rnn_thr)
    voltage_reg_rnn = tf.reduce_sum(tf.reduce_mean(tf.square(rnn_pos), 1))
    voltage_reg_rnn += tf.reduce_sum(tf.reduce_mean(tf.square(rnn_neg), 1))
    rnn_rate = tf.reduce_mean(custom_rnn_output[0], (0, 1))
    rnn_mean_rate = tf.reduce_mean(rnn_rate)
    analysis_tensors['rnn_rate'] = rnn_mean_rate
    rate_loss = tf.reduce_sum(tf.square(rnn_rate - .02)) * 1.
    torso_from_collection = tf.get_collection('torso_output')[-1]

    conv1_z = torso_from_collection['c1_z']
    conv2_z = torso_from_collection['c2_z']
    lin_z = torso_from_collection['lin_z']
    conv_1_rate = tf.reduce_mean(conv1_z, (0, 1))
    conv_2_rate = tf.reduce_mean(conv2_z, (0, 1))
    linear_rate = tf.reduce_mean(lin_z, (0, 1))
    mean_conv_1_rate = tf.reduce_mean(conv_1_rate)
    mean_conv_2_rate = tf.reduce_mean(conv_2_rate)
    mean_linear_rate = tf.reduce_mean(linear_rate)
    analysis_tensors['conv1_z'] = conv1_z
    analysis_tensors['conv2_z'] = conv2_z
    analysis_tensors['conv1_rate'] = mean_conv_1_rate
    analysis_tensors['conv2_rate'] = mean_conv_2_rate
    analysis_tensors['linear_rate'] = mean_linear_rate
    conv1_v = torso_from_collection['c1_act']
    conv2_v = torso_from_collection['c2_act']
    analysis_tensors['conv1_v'] = conv1_v
    analysis_tensors['conv2_v'] = conv2_v
    conv_pos = tf.nn.relu(conv1_v - FLAGS.thr_scnn)
    conv_neg = tf.nn.relu(-conv1_v - FLAGS.thr_scnn)
    if FLAGS.voltage_reg_method == 'avg_all':
        voltage_reg = tf.reduce_sum(tf.square(tf.reduce_mean(conv_pos,
                                                             (0, 1))))
        voltage_reg += tf.reduce_sum(
            tf.square(tf.reduce_mean(conv_neg, (0, 1))))
    elif FLAGS.voltage_reg_method == 'avg_time':
        voltage_reg = tf.reduce_sum(
            tf.reduce_mean(tf.square(tf.reduce_mean(conv_pos, 1)), 0))
        voltage_reg += tf.reduce_sum(
            tf.reduce_mean(tf.square(tf.reduce_mean(conv_neg, 1)), 0))
    conv_pos = tf.nn.relu(conv2_v - FLAGS.thr_scnn)
    conv_neg = tf.nn.relu(-conv2_v - FLAGS.thr_scnn)
    if FLAGS.voltage_reg_method == 'avg_all':
        voltage_reg += tf.reduce_sum(
            tf.square(tf.reduce_mean(conv_pos, (0, 1))))
        voltage_reg += tf.reduce_sum(
            tf.square(tf.reduce_mean(conv_neg, (0, 1))))
    elif FLAGS.voltage_reg_method == 'avg_time':
        voltage_reg += tf.reduce_sum(
            tf.reduce_mean(tf.square(tf.reduce_mean(conv_pos, 1)), 0))
        voltage_reg += tf.reduce_sum(
            tf.reduce_mean(tf.square(tf.reduce_mean(conv_neg, 1)), 0))
    reg_loss = rate_loss * FLAGS.rate_cost
    reg_loss += FLAGS.voltage_cost_rnn * voltage_reg_rnn
    reg_loss += FLAGS.voltage_cost_cnn * voltage_reg

    bootstrap_value = learner_outputs.baseline[-1]

    agent_outputs = nest.map_structure(lambda t: t[1:], agent_outputs)
    rewards = env_outputs.reward[1:]
    infos = nest.map_structure(lambda t: t[1:], env_outputs.info)
    done = to_bool(tf.cast(env_outputs.done, tf.int64)[1:])
    learner_outputs = nest.map_structure(lambda t: t[:-1], learner_outputs)

    discounts = tf.to_float(~done) * FLAGS.discounting

    def scan_fun(_accumulator, _input):
        _discount, _reward = _input
        return _reward + _discount * _accumulator

    value_targets = tf.stop_gradient(
        tf.scan(scan_fun, (discounts, rewards),
                initializer=bootstrap_value,
                reverse=True))
    adv = value_targets - learner_outputs.baseline

    shifted_done = tf.concat(
        (tf.cast(done[1:], tf.float32), tf.zeros((1, tf.shape(done)[1]))), 0)

    if episode_finished_mask is not None:
        finished_mask = (1 - tf.cumsum(shifted_done, 0)) * (
            1 - tf.cast(episode_finished_mask, tf.float32))[None]
    else:
        finished_mask = None

    pg_loss = compute_policy_gradient_loss(learner_outputs.policy_logits,
                                           agent_outputs.action,
                                           adv,
                                           mask=finished_mask)
    value_loss = FLAGS.baseline_cost * compute_baseline_loss(
        value_targets - learner_outputs.baseline, mask=finished_mask)
    entropy_loss = FLAGS.entropy_cost * compute_entropy_loss(
        learner_outputs.policy_logits, mask=finished_mask)
    p_actions = tf.nn.softmax(learner_outputs.policy_logits)

    loss_per_timestep = pg_loss + value_loss + entropy_loss
    total_loss = tf.reduce_sum(loss_per_timestep)
    total_loss += reg_loss

    total_loss_for_cnn = tf.reduce_sum(
        pg_loss + value_loss) / FLAGS.reg_factor_cnn + reg_loss

    num_env_frames = tf.train.get_global_step()
    optimizer = tf.train.AdamOptimizer(learning_rate, epsilon=FLAGS.epsilon)
    cnn_optimizer = tf.train.AdamOptimizer(learning_rate,
                                           epsilon=FLAGS.epsilon)
    normal_grad_norm = tf.zeros(())
    elig_grad_norm = tf.zeros(())
    rl_elig_grad_norm = tf.zeros(())
    recurrent_grads_and_vars = optimizer.compute_gradients(
        total_loss, var_list=recurrent_variables)
    if FLAGS.ba_cnn or FLAGS.avg_ba_cnn:
        torso_grads_and_vars = cnn_optimizer.compute_gradients(
            total_loss_for_cnn, var_list=torso_variables)
    else:
        torso_grads_and_vars = cnn_optimizer.compute_gradients(
            total_loss, var_list=torso_variables)
    grads_and_vars = [*recurrent_grads_and_vars, *torso_grads_and_vars]
    eligibility_variables = agent.core.variable_list

    for i, (g, v) in enumerate(grads_and_vars):
        for ve in eligibility_variables:
            if ve == v:
                normal_grad_norm += tf.reduce_sum(tf.square(g))
    if gradients_buffer is not None:
        add_ops = []
        for g, v in grads_and_vars:
            for g_holder, v_holder in gradients_buffer:
                if v == v_holder:
                    add_ops.append(tf.assign_add(g_holder, g))
        train_op = tf.group(*add_ops)
    else:
        train_op = tf.group(
            optimizer.apply_gradients(recurrent_grads_and_vars),
            cnn_optimizer.apply_gradients(torso_grads_and_vars))

    op = tf.no_op()
    with tf.control_dependencies([train_op, op]):
        num_env_frames_and_train = num_env_frames.assign_add(
            FLAGS.parallel_episodes * FLAGS.unroll_length *
            FLAGS.num_action_repeats)

    analysis_tensors['normal_grad_norm'] = normal_grad_norm
    analysis_tensors['elig_grad_norm'] = elig_grad_norm
    analysis_tensors['rl_elig_grad_norm'] = rl_elig_grad_norm
    analysis_tensors['pg_loss'] = pg_loss
    analysis_tensors['value_loss'] = value_loss
    analysis_tensors['entropy_loss'] = entropy_loss
    analysis_tensors['rate_loss'] = rate_loss * FLAGS.rate_cost
    analysis_tensors[
        'voltage_loss_rnn'] = voltage_reg_rnn * FLAGS.voltage_cost_rnn
    analysis_tensors['voltage_loss_cnn'] = voltage_reg * FLAGS.voltage_cost_cnn
    analysis_tensors['action_distribution'] = p_actions
    analysis_tensors['value_target'] = value_targets

    return (done, infos, num_env_frames_and_train,
            analysis_tensors), optimizer, grads_and_vars
Exemplo n.º 39
0
 def set(self, cursor=None, facebook_id=None, twitter_token=None,
         facebook=None, twitter=None, forename=None, surname=None, age=None,
         birth_day=None, birth_month=None, birth_year=None, gender=None,
         employee=None, country=None, language=None, email=None, top5=None,
         twitter_id=None, twitter_secret=None, save_locally=None,
         app_version=None, iphone_model=None, ios_version=None,
         last_facebook=None, last_twitter=None, **kwargs):
     if not facebook_id:
         raise cherrypy.HTTPError(403)
     qry = {'select':   'COUNT(id) AS count',
            'table':    'users',
            'where':    'facebook_id = ?'}
     cursor.execute(util.query(**qry), (facebook_id,))
     res = cursor.fetchone().count
     data = {'twitter_id':     twitter_id,
             'twitter_token':  twitter_token,
             'twitter_secret': twitter_secret,
             'facebook':       facebook,
             'twitter':        twitter,
             'forename':       forename,
             'surname':        surname,
             'age':            util.to_int(age),
             'birth_day':      util.to_int(birth_day),
             'birth_month':    util.to_int(birth_month),
             'birth_year':     util.to_int(birth_year),
             'gender':         gender,
             'employee':       util.to_bool(employee),
             'country':        country,
             'language':       language,
             'email':          email,
             'top5':           util.to_bool(top5),
             'save_locally':   util.to_bool(save_locally),
             'app_version':    app_version,
             'iphone_model':   iphone_model,
             'ios_version':    ios_version,
             'last_login':     util.now(),
             'last_facebook':  util.to_bool(last_facebook),
             'last_twitter':   util.to_bool(last_twitter)}
     columns = []
     values = []
     for key, val in data.iteritems():
         if val != None:
             columns.append(key)
             if val is not True and val is not False:
                 values.append(val)
             else:
                 if val:
                     values.append('1')
                 else:
                     values.append('0')
     values.append(facebook_id)
     if res:
         qry = {'update':     'users',
                'set_values': columns,
                'where':      'facebook_id = ?'}
         cursor.execute(util.query(**qry), values)
     else:
         columns.append('facebook_id')
         columns.append('joined')
         values.append(util.now())
         qry = {'insert_into': 'users',
                'columns':     columns}
         cursor.execute(util.query(**qry), values)
     qry = {'select':   'id',
            'table':    'users',
            'where':    'facebook_id = ?',
            'order_by': 'id',
            'limit':    1}
     cursor.execute(util.query(**qry), (facebook_id,))
     user_id = cursor.fetchone().id
     return self.retrieve(cursor=cursor, user_id=user_id)
Exemplo n.º 40
0
 def is_locked(self, scope, guid):
     return util.to_bool(
         execute_scalar(scope.get_handle(),
                        "select locked from Object where guid=%s", guid))
Exemplo n.º 41
0
	def __post__(self, env, email, firstname, lastname, gender, language, protected):
		self.app.update_user_details(self.username, email, firstname, lastname, gender, language, util.to_bool(protected))

		v = view.JSONView(200)
		m = self.app.get_full_user_details(self.username)
		v.bind(m)

		return v