Example #1
0
def weblog_entry_dict(logline, log_entry_modelcls):
    entry = pattern_grok.match(logline)
    if entry:
        entry = subdict(entry, column_names(log_entry_modelcls))
        if entry['session_id'] == 'None':
            entry['session_id'] = None
        if entry['user_id'] == '_ANON_None':
            entry['user_id'] = None
        for k, v in entry.items():
            if v == "-":
                entry[k] = None

        timestamp_format = '%d/%b/%Y:%H:%M:%S %z'
        entry['created_on'] = datetime.strptime(entry['created_on'],
                                                timestamp_format)
        for k in ['created_on', 'remote_address', 'method', 'request_uri']:
            if entry.get(k) is None:
                return None
        for field, _type in field_types.items():
            entry[field] = null_safe_type_cast(_type, entry[field])

        entry['raw_text'] = logline
        for col in column_names(log_entry_modelcls):
            if col not in entry and col != 'id':
                entry[col] = None
    return entry
Example #2
0
    def find_or_create_all(cls, list_of_kwargs, keys=[]):
        """Batch method for querying for a list of instances and
        creating them if required

        Args:
            list_of_kwargs(list of dicts): A list of dicts where
                each dict denotes the keyword args that you would pass
                to the create method separately

            keys (list, optional): A list of keys to use for the 
                initial finding step. Matching is done only on these
                attributes.

        Examples:

            >>> Customer.find_or_create_all([
            ... {'name': 'Vicky', 'email': '*****@*****.**', 'age': 34},
            ... {'name': 'Ron', 'age': 40, 'email': '*****@*****.**',
            ... 'gender': 'Male'}], keys=['name', 'email'])
        """
        list_of_kwargs_wo_dupes, markers = remove_and_mark_duplicate_dicts(
            list_of_kwargs, keys)
        added_objs = cls.add_all([
            cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
            for kwargs in list_of_kwargs_wo_dupes
        ])
        result_objs = []
        iterator_of_added_objs = iter(added_objs)
        for idx in range(len(list_of_kwargs)):
            if idx in markers:
                result_objs.append(added_objs[markers[idx]])
            else:
                result_objs.append(next(iterator_of_added_objs))
        return result_objs
    def new(cls, **kwargs):
        """Returns a new, unsaved instance of the model class.

        """
        kwargs = cls.preprocess_kwargs_before_new(kwargs)
        if cls.__mapper__.polymorphic_on is not None:
            discriminator_key = cls.__mapper__.polymorphic_on.name
            discriminator_val = kwargs.get(discriminator_key)
            if discriminator_val is not None and discriminator_val in cls.__mapper__.polymorphic_map:
                actual_cls = cls.__mapper__.polymorphic_map[discriminator_val].class_
                return actual_cls(
                    **subdict(
                        actual_cls._preprocess_params(kwargs),
                        actual_cls.all_settable_keys())
                )
        return cls(**subdict(cls._preprocess_params(kwargs), cls.all_settable_keys()))
    def find_or_create_all(cls, list_of_kwargs, keys=[]):
        """Batch method for querying for a list of instances and
        creating them if required

        Args:
            list_of_kwargs(list of dicts): A list of dicts where
                each dict denotes the keyword args that you would pass
                to the create method separately

            keys (list, optional): A list of keys to use for the 
                initial finding step. Matching is done only on these
                attributes.

        Examples:

            >>> Customer.find_or_create_all([
            ... {'name': 'Vicky', 'email': '*****@*****.**', 'age': 34},
            ... {'name': 'Ron', 'age': 40, 'email': '*****@*****.**',
            ... 'gender': 'Male'}], keys=['name', 'email'])
        """
        list_of_kwargs_wo_dupes, markers = remove_and_mark_duplicate_dicts(
            list_of_kwargs, keys)
        added_objs = cls.add_all([
            cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
            for kwargs in list_of_kwargs_wo_dupes])
        result_objs = []
        iterator_of_added_objs = iter(added_objs)
        for idx in range(len(list_of_kwargs)):
            if idx in markers:
                result_objs.append(added_objs[markers[idx]])
            else:
                result_objs.append(next(
                    iterator_of_added_objs))
        return result_objs
Example #5
0
    def new(cls, **kwargs):
        """Returns a new, unsaved instance of the model class.

        """
        kwargs = cls.pre_save_adapter(kwargs)
        if cls.__mapper__.polymorphic_on is not None:
            discriminator_key = cls.__mapper__.polymorphic_on.name
            discriminator_val = kwargs.get(discriminator_key)
            if discriminator_val is not None and discriminator_val in cls.__mapper__.polymorphic_map:
                actual_cls = cls.__mapper__.polymorphic_map[
                    discriminator_val].class_
                return actual_cls(
                    **subdict(actual_cls._prepare_data_for_saving(kwargs),
                              actual_cls.all_settable_keys()))
        return cls(**subdict(cls._prepare_data_for_saving(kwargs),
                             cls.all_settable_keys()))
Example #6
0
    def find_or_create(cls, **kwargs):
        """Checks if an instance already exists in db with these kwargs else
        returns a new, saved instance of the service's model class.

        :param **kwargs: instance parameters
        """
        keys = kwargs.pop('keys') if 'keys' in kwargs else []
        return cls.first(**subdict(kwargs, keys)) or cls.create(**kwargs)
    def find_or_build(cls, **kwargs):
        """Checks if an instance already exists in db with these kwargs else
        returns a new, saved instance of the service's model class.

        Args:
            **kwargs: instance parameters
        """
        keys = kwargs.pop('keys') if 'keys' in kwargs else []
        return cls.first(**subdict(kwargs, keys)) or cls.build(**kwargs)
    def update_or_create(cls, **kwargs):
        """Checks if an instance already exists by filtering with the
        kwargs. If yes, updates the instance with new kwargs and
        returns that instance. If not, creates a new
        instance with kwargs and returns it.

        Args:

            **kwargs: The keyword arguments which are used for filtering
                and initialization.

            keys (list, optional): A special keyword argument. If passed,
                only the set of keys mentioned here will be used for filtering.
                Useful when we want to 'filter' based on a subset of the keys
                and create with all the keys.

        Examples:

            >>> customer = Customer.update_or_create(
            ...     name="vicky", email="*****@*****.**", country="India")
            >>> customer.id
            45
            >>> customer1 = Customer.update_or_create(
            ...     name="vicky", email="*****@*****.**", country="India")
            >>> customer1==customer
            True
            >>> customer2 = Customer.update_or_create(
            ...     name="vicky", email="*****@*****.**", country="Russia")
            >>> customer2==customer
            False
            >>> customer3 = Customer.update_or_create(
            ...      name="vicky", email="*****@*****.**", country="Russia",
            ...      keys=['name', 'email'])
            >>> customer3==customer
            True
        """
        keys = kwargs.pop('keys') if 'keys' in kwargs else []
        filter_kwargs = subdict(kwargs, keys)
        if filter_kwargs == {}:
            obj = None
        else:
            obj = cls.first(**filter_kwargs)
        if obj is not None:
            for key, value in kwargs.iteritems():
                if (key not in keys and
                        key not in cls._no_overwrite_):
                    setattr(obj, key, value)
            try:
                cls.session.commit()
            except:
                cls.session.rollback()
                raise
        else:
            obj = cls.create(**kwargs)
        return obj
Example #9
0
    def update_or_create(cls, **kwargs):
        """Checks if an instance already exists by filtering with the
        kwargs. If yes, updates the instance with new kwargs and
        returns that instance. If not, creates a new
        instance with kwargs and returns it.

        Args:

            **kwargs: The keyword arguments which are used for filtering
                and initialization.

            keys (list, optional): A special keyword argument. If passed,
                only the set of keys mentioned here will be used for filtering.
                Useful when we want to 'filter' based on a subset of the keys
                and create with all the keys.

        Examples:

            >>> customer = Customer.update_or_create(
            ...     name="vicky", email="*****@*****.**", country="India")
            >>> customer.id
            45
            >>> customer1 = Customer.update_or_create(
            ...     name="vicky", email="*****@*****.**", country="India")
            >>> customer1==customer
            True
            >>> customer2 = Customer.update_or_create(
            ...     name="vicky", email="*****@*****.**", country="Russia")
            >>> customer2==customer
            False
            >>> customer3 = Customer.update_or_create(
            ...      name="vicky", email="*****@*****.**", country="Russia",
            ...      keys=['name', 'email'])
            >>> customer3==customer
            True
        """
        keys = kwargs.pop('keys') if 'keys' in kwargs else []
        filter_kwargs = subdict(kwargs, keys)
        if filter_kwargs == {}:
            obj = None
        else:
            obj = cls.first(**filter_kwargs)
        if obj is not None:
            for key, value in six.iteritems(kwargs):
                if (key not in keys and key not in cls._no_overwrite_):
                    setattr(obj, key, value)
            try:
                cls.session.commit()
            except:
                cls.session.rollback()
                raise
        else:
            obj = cls.create(**kwargs)
        return obj
Example #10
0
 def update_or_build(cls, **kwargs):
     keys = kwargs.pop('keys') if 'keys' in kwargs else []
     filter_kwargs = subdict(kwargs, keys)
     if filter_kwargs == {}:
         obj = None
     else:
         obj = cls.first(**filter_kwargs)
     if obj is not None:
         for key, value in six.iteritems(kwargs):
             if (key not in keys and key not in cls._no_overwrite_):
                 setattr(obj, key, value)
     else:
         obj = cls.build(**kwargs)
     return obj
 def update_or_build(cls, **kwargs):
     keys = kwargs.pop('keys') if 'keys' in kwargs else []
     filter_kwargs = subdict(kwargs, keys)
     if filter_kwargs == {}:
         obj = None
     else:
         obj = cls.first(**filter_kwargs)
     if obj is not None:
         for key, value in kwargs.iteritems():
             if (key not in keys and
                     key not in cls._no_overwrite_):
                 setattr(obj, key, value)
     else:
         obj = cls.build(**kwargs)
     return obj
Example #12
0
 def update_or_create(cls, **kwargs):
     keys = kwargs.pop('keys') if 'keys' in kwargs else []
     obj = cls.first(**subdict(kwargs, keys))
     if obj is not None:
         for key, value in kwargs.iteritems():
             if (key not in keys and key not in cls.__no_overwrite__):
                 setattr(obj, key, value)
         try:
             cls.session.commit()
         except:
             cls.session.rollback()
             raise
     else:
         obj = cls.create(**kwargs)
     return obj
Example #13
0
 def find_or_create_all(cls, list_of_kwargs, keys=[]):
     list_of_kwargs_wo_dupes, markers = remove_and_mark_duplicate_dicts(
         list_of_kwargs, keys)
     added_objs = cls.add_all([
         cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
         for kwargs in list_of_kwargs_wo_dupes
     ])
     result_objs = []
     iterator_of_added_objs = iter(added_objs)
     for idx in range(len(list_of_kwargs)):
         if idx in markers:
             result_objs.append(added_objs[markers[idx]])
         else:
             result_objs.append(next(iterator_of_added_objs))
     return result_objs
Example #14
0
 def update_or_create_all(cls, list_of_kwargs, keys=[]):
     objs = []
     for kwargs in list_of_kwargs:
         obj = cls.first(**subdict(kwargs, keys))
         if obj is not None:
             for key, value in kwargs.iteritems():
                 if (key not in keys and key not in cls.__no_overwrite__):
                     setattr(obj, key, value)
         else:
             obj = cls.new(**kwargs)
         objs.append(obj)
     try:
         return cls.add_all(objs)
     except:
         cls.session.rollback()
         raise
    def update_or_build_all(cls, list_of_kwargs, keys=[]):
        """Batch method for updating a list of instances and
        creating them if required

        Args:
            list_of_kwargs(list of dicts): A list of dicts where
                each dict denotes the keyword args that you would pass
                to the create method separately

            keys (list, optional): A list of keys to use for the 
                initial finding step. Matching is done only on these
                attributes.

        Examples:

            >>> Customer.update_or_create_all([
            ... {'name': 'Vicky', 'email': '*****@*****.**', 'age': 34},
            ... {'name': 'Ron', 'age': 40, 'email': '*****@*****.**',
            ... 'gender': 'Male'}], keys=['name', 'email'])
        """
        objs = []
        for kwargs in list_of_kwargs:
            filter_kwargs = subdict(kwargs, keys)
            if filter_kwargs == {}:
                obj = None
            else:
                obj = cls.first(**filter_kwargs)
            if obj is not None:
                for key, value in kwargs.iteritems():
                    if (key not in keys and
                            key not in cls._no_overwrite_):
                        setattr(obj, key, value)
            else:
                obj = cls.new(**kwargs)
            objs.append(obj)
        try:
            return cls.add_all(objs, commit=False)
        except:
            cls.session.rollback()
            raise
Example #16
0
    def update_or_build_all(cls, list_of_kwargs, keys=[]):
        """Batch method for updating a list of instances and
        creating them if required

        Args:
            list_of_kwargs(list of dicts): A list of dicts where
                each dict denotes the keyword args that you would pass
                to the create method separately

            keys (list, optional): A list of keys to use for the 
                initial finding step. Matching is done only on these
                attributes.

        Examples:

            >>> Customer.update_or_create_all([
            ... {'name': 'Vicky', 'email': '*****@*****.**', 'age': 34},
            ... {'name': 'Ron', 'age': 40, 'email': '*****@*****.**',
            ... 'gender': 'Male'}], keys=['name', 'email'])
        """
        objs = []
        for kwargs in list_of_kwargs:
            filter_kwargs = subdict(kwargs, keys)
            if filter_kwargs == {}:
                obj = None
            else:
                obj = cls.first(**filter_kwargs)
            if obj is not None:
                for key, value in six.iteritems(kwargs):
                    if (key not in keys and key not in cls._no_overwrite_):
                        setattr(obj, key, value)
            else:
                obj = cls.new(**kwargs)
            objs.append(obj)
        try:
            return cls.add_all(objs, commit=False)
        except:
            cls.session.rollback()
            raise
 def get_updated_or_new_obj(cls, kwargs=None, filter_keys=None):
     if filter_keys is None:
         filter_keys = []
     if kwargs is None:
         kwargs = {}
     filter_kwargs = subdict(kwargs, filter_keys)
     if filter_kwargs == {}:
         obj = None
     else:
         obj = cls.first(**filter_kwargs)
     if obj is not None:
         update_kwargs = {
             k: v for k, v in kwargs.iteritems()
             if k not in filter_keys and k not in cls._no_overwrite_}
         obj.update_without_commit(**update_kwargs)
         # for key, value in kwargs.iteritems():
         #     if (key not in filter_keys and
         #             key not in cls._no_overwrite_):
         #         setattr(obj, key, value)
     else:
         obj = cls.new(**kwargs)
     return obj
def construct_query_modifiers(query_modifiers=None,
                              allow_modification_via_requests=True):
    default_query_modifiers = {
        "page": None,
        "per_page": 20,
        "limit": None,
        "offset": None,
        "order_by": None,
        "sort": "asc",
    }
    if query_modifiers is None:
        query_modifiers = {}
    query_modifiers = subdict(query_modifiers, QUERY_MODIFIERS)
    query_modifiers = merge(default_query_modifiers, query_modifiers)
    if allow_modification_via_requests:
        query_modifiers = merge(query_modifiers,
                                fetch_query_modifiers_from_request())
    for k in ['page', 'per_page', 'limit', 'offset']:
        if k in query_modifiers:
            query_modifiers[k] = null_safe_type_cast(int,
                                                     query_modifiers.get(k))
    return query_modifiers
    def find_or_create(cls, **kwargs):
        """Checks if an instance already exists by filtering with the
        kwargs. If yes, returns that instance. If not, creates a new
        instance with kwargs and returns it

        Args:

            **kwargs: The keyword arguments which are used for filtering
                and initialization.
            keys(list, optional): A special keyword argument.
                If passed, only the set of keys mentioned here will be used
                for filtering. Useful when we want to 'find' based on a subset
                of the keys and create with all the keys

        Examples:

            >>> customer = Customer.find_or_create(
            ...     name="vicky", email="*****@*****.**", country="India")
            >>> customer.id
            45
            >>> customer1 = Customer.find_or_create(
            ...     name="vicky", email="*****@*****.**", country="India")
            >>> customer1==customer
            True
            >>> customer2 = Customer.find_or_create(
            ...     name="vicky", email="*****@*****.**", country="Russia")
            >>> customer2==customer
            False
            >>> customer3 = Customer.find_or_create(
            ...      name="vicky", email="*****@*****.**", country="Russia",
            ...      keys=['name', 'email'])
            >>> customer3==customer
            True
        """
        keys = kwargs.pop('keys') if 'keys' in kwargs else []
        return cls.first(**subdict(kwargs, keys)) or cls.create(**kwargs)
Example #20
0
 def get_updated_or_new_obj(cls, kwargs=None, filter_keys=None):
     if filter_keys is None:
         filter_keys = []
     if kwargs is None:
         kwargs = {}
     filter_kwargs = subdict(kwargs, filter_keys)
     if filter_kwargs == {}:
         obj = None
     else:
         obj = cls.first(**filter_kwargs)
     if obj is not None:
         update_kwargs = {
             k: v
             for k, v in six.iteritems(kwargs)
             if k not in filter_keys and k not in cls._no_overwrite_
         }
         obj.update_without_commit(**update_kwargs)
         # for key, value in kwargs.iteritems():
         #     if (key not in filter_keys and
         #             key not in cls._no_overwrite_):
         #         setattr(obj, key, value)
     else:
         obj = cls.new(**kwargs)
     return obj
Example #21
0
    def find_or_create(cls, **kwargs):
        """Checks if an instance already exists by filtering with the
        kwargs. If yes, returns that instance. If not, creates a new
        instance with kwargs and returns it

        Args:

            **kwargs: The keyword arguments which are used for filtering
                and initialization.
            keys(list, optional): A special keyword argument.
                If passed, only the set of keys mentioned here will be used
                for filtering. Useful when we want to 'find' based on a subset
                of the keys and create with all the keys

        Examples:

            >>> customer = Customer.find_or_create(
            ...     name="vicky", email="*****@*****.**", country="India")
            >>> customer.id
            45
            >>> customer1 = Customer.find_or_create(
            ...     name="vicky", email="*****@*****.**", country="India")
            >>> customer1==customer
            True
            >>> customer2 = Customer.find_or_create(
            ...     name="vicky", email="*****@*****.**", country="Russia")
            >>> customer2==customer
            False
            >>> customer3 = Customer.find_or_create(
            ...      name="vicky", email="*****@*****.**", country="Russia",
            ...      keys=['name', 'email'])
            >>> customer3==customer
            True
        """
        keys = kwargs.pop('keys') if 'keys' in kwargs else []
        return cls.first(**subdict(kwargs, keys)) or cls.create(**kwargs)
Example #22
0
 def find_or_new(cls, **kwargs):
     keys = kwargs.pop('keys') if 'keys' in kwargs else []
     return cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
Example #23
0
        ([['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XL-INCH', 1],
          ['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
          ['U326-CDCHFLGOSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
          ['U326-NMSKU-SNACKDOWNBADGE', 1]],
         [['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XL-INCH', 1],
          ['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
          ['U326-RFRECHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
          ['U326-NMSKU-SNACKDOWNBADGE', 1]]),
        'XXL':
        [['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XXL-INCH', 1],
         ['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
         ['U326-PRTECHFSTCKR-V14-ST-DCU-3X3D-C7FC-OP-3X3-INCH', 1],
         ['U326-NMSKU-SNACKDOWNBADGE', 1]],
        'XXXL':
        [['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XXXL-INCH', 1],
         ['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
         ['U326-PRTECHFSTCKR-V14-ST-DCU-3X3D-C7FC-OP-3X3-INCH', 1],
         ['U326-NMSKU-SNACKDOWNBADGE', 1]]
    }
    shipments = create_shipments(skus=skus,
                                 recipients=recipients,
                                 key=key,
                                 secret=secret)
    print "Read %s rows from the sheet" % sum(
        len(v) for v in recipients.values())
    print "Split up to send"
    print "Found %s valid entries for shipments" % sum(
        len(v) for v in subdict(recipients, skus.keys()).values())
    print "Created following %s shipments:" % len(shipments)
    print shipments
def fetch_query_modifiers_from_request():
    return subdict(request.args, QUERY_MODIFIERS)
 def find_or_new(cls, **kwargs):
     keys = kwargs.pop('keys') if 'keys' in kwargs else []
     return cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
            [['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XL-INCH', 1],
             ['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
             ['U326-CDCHFLGOSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
             ['U326-NMSKU-SNACKDOWNBADGE', 1]],

            [['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XL-INCH', 1],
             ['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
             ['U326-RFRECHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
             ['U326-NMSKU-SNACKDOWNBADGE', 1]]
        ),

        'XXL': [['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XXL-INCH', 1],
                ['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
                ['U326-PRTECHFSTCKR-V14-ST-DCU-3X3D-C7FC-OP-3X3-INCH', 1],
                ['U326-NMSKU-SNACKDOWNBADGE', 1]],

        'XXXL': [['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XXXL-INCH', 1],
                 ['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
                 ['U326-PRTECHFSTCKR-V14-ST-DCU-3X3D-C7FC-OP-3X3-INCH', 1],
                 ['U326-NMSKU-SNACKDOWNBADGE', 1]]
    }
    shipments = create_shipments(
        skus=skus, recipients=recipients, key=key, secret=secret)
    print "Read %s rows from the sheet" % sum(
        len(v) for v in recipients.values())
    print "Split up to send"
    print "Found %s valid entries for shipments" % sum(
        len(v) for v in subdict(recipients, skus.keys()).values())
    print "Created following %s shipments:" % len(shipments)
    print shipments
 def find_or_new_all(cls, list_of_kwargs, keys=[]):
     return [cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs) for kwargs in list_of_kwargs]
Example #28
0
 def find_or_new_all(cls, list_of_kwargs, keys=[]):
     return [
         cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
         for kwargs in list_of_kwargs
     ]