def weblog_entry_dict(logline, log_entry_modelcls):
entry = pattern_grok.match(logline)
if entry:
entry = subdict(entry, column_names(log_entry_modelcls))
if entry['session_id'] == 'None':
entry['session_id'] = None
if entry['user_id'] == '_ANON_None':
entry['user_id'] = None
for k, v in entry.items():
if v == "-":
entry[k] = None
timestamp_format = '%d/%b/%Y:%H:%M:%S %z'
entry['created_on'] = datetime.strptime(entry['created_on'],
timestamp_format)
for k in ['created_on', 'remote_address', 'method', 'request_uri']:
if entry.get(k) is None:
return None
for field, _type in field_types.items():
entry[field] = null_safe_type_cast(_type, entry[field])
entry['raw_text'] = logline
for col in column_names(log_entry_modelcls):
if col not in entry and col != 'id':
entry[col] = None
return entry
def find_or_create_all(cls, list_of_kwargs, keys=[]):
"""Batch method for querying for a list of instances and
creating them if required
Args:
list_of_kwargs(list of dicts): A list of dicts where
each dict denotes the keyword args that you would pass
to the create method separately
keys (list, optional): A list of keys to use for the
initial finding step. Matching is done only on these
attributes.
Examples:
>>> Customer.find_or_create_all([
... {'name': 'Vicky', 'email': '*****@*****.**', 'age': 34},
... {'name': 'Ron', 'age': 40, 'email': '*****@*****.**',
... 'gender': 'Male'}], keys=['name', 'email'])
"""
list_of_kwargs_wo_dupes, markers = remove_and_mark_duplicate_dicts(
list_of_kwargs, keys)
added_objs = cls.add_all([
cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
for kwargs in list_of_kwargs_wo_dupes
])
result_objs = []
iterator_of_added_objs = iter(added_objs)
for idx in range(len(list_of_kwargs)):
if idx in markers:
result_objs.append(added_objs[markers[idx]])
else:
result_objs.append(next(iterator_of_added_objs))
return result_objs
def new(cls, **kwargs):
"""Returns a new, unsaved instance of the model class.
"""
kwargs = cls.preprocess_kwargs_before_new(kwargs)
if cls.__mapper__.polymorphic_on is not None:
discriminator_key = cls.__mapper__.polymorphic_on.name
discriminator_val = kwargs.get(discriminator_key)
if discriminator_val is not None and discriminator_val in cls.__mapper__.polymorphic_map:
actual_cls = cls.__mapper__.polymorphic_map[discriminator_val].class_
return actual_cls(
**subdict(
actual_cls._preprocess_params(kwargs),
actual_cls.all_settable_keys())
)
return cls(**subdict(cls._preprocess_params(kwargs), cls.all_settable_keys()))
def find_or_create_all(cls, list_of_kwargs, keys=[]):
"""Batch method for querying for a list of instances and
creating them if required
Args:
list_of_kwargs(list of dicts): A list of dicts where
each dict denotes the keyword args that you would pass
to the create method separately
keys (list, optional): A list of keys to use for the
initial finding step. Matching is done only on these
attributes.
Examples:
>>> Customer.find_or_create_all([
... {'name': 'Vicky', 'email': '*****@*****.**', 'age': 34},
... {'name': 'Ron', 'age': 40, 'email': '*****@*****.**',
... 'gender': 'Male'}], keys=['name', 'email'])
"""
list_of_kwargs_wo_dupes, markers = remove_and_mark_duplicate_dicts(
list_of_kwargs, keys)
added_objs = cls.add_all([
cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
for kwargs in list_of_kwargs_wo_dupes])
result_objs = []
iterator_of_added_objs = iter(added_objs)
for idx in range(len(list_of_kwargs)):
if idx in markers:
result_objs.append(added_objs[markers[idx]])
else:
result_objs.append(next(
iterator_of_added_objs))
return result_objs
def new(cls, **kwargs):
"""Returns a new, unsaved instance of the model class.
"""
kwargs = cls.pre_save_adapter(kwargs)
if cls.__mapper__.polymorphic_on is not None:
discriminator_key = cls.__mapper__.polymorphic_on.name
discriminator_val = kwargs.get(discriminator_key)
if discriminator_val is not None and discriminator_val in cls.__mapper__.polymorphic_map:
actual_cls = cls.__mapper__.polymorphic_map[
discriminator_val].class_
return actual_cls(
**subdict(actual_cls._prepare_data_for_saving(kwargs),
actual_cls.all_settable_keys()))
return cls(**subdict(cls._prepare_data_for_saving(kwargs),
cls.all_settable_keys()))
def find_or_create(cls, **kwargs):
"""Checks if an instance already exists in db with these kwargs else
returns a new, saved instance of the service's model class.
:param **kwargs: instance parameters
"""
keys = kwargs.pop('keys') if 'keys' in kwargs else []
return cls.first(**subdict(kwargs, keys)) or cls.create(**kwargs)
def find_or_build(cls, **kwargs):
"""Checks if an instance already exists in db with these kwargs else
returns a new, saved instance of the service's model class.
Args:
**kwargs: instance parameters
"""
keys = kwargs.pop('keys') if 'keys' in kwargs else []
return cls.first(**subdict(kwargs, keys)) or cls.build(**kwargs)
def update_or_create(cls, **kwargs):
"""Checks if an instance already exists by filtering with the
kwargs. If yes, updates the instance with new kwargs and
returns that instance. If not, creates a new
instance with kwargs and returns it.
Args:
**kwargs: The keyword arguments which are used for filtering
and initialization.
keys (list, optional): A special keyword argument. If passed,
only the set of keys mentioned here will be used for filtering.
Useful when we want to 'filter' based on a subset of the keys
and create with all the keys.
Examples:
>>> customer = Customer.update_or_create(
... name="vicky", email="*****@*****.**", country="India")
>>> customer.id
45
>>> customer1 = Customer.update_or_create(
... name="vicky", email="*****@*****.**", country="India")
>>> customer1==customer
True
>>> customer2 = Customer.update_or_create(
... name="vicky", email="*****@*****.**", country="Russia")
>>> customer2==customer
False
>>> customer3 = Customer.update_or_create(
... name="vicky", email="*****@*****.**", country="Russia",
... keys=['name', 'email'])
>>> customer3==customer
True
"""
keys = kwargs.pop('keys') if 'keys' in kwargs else []
filter_kwargs = subdict(kwargs, keys)
if filter_kwargs == {}:
obj = None
else:
obj = cls.first(**filter_kwargs)
if obj is not None:
for key, value in kwargs.iteritems():
if (key not in keys and
key not in cls._no_overwrite_):
setattr(obj, key, value)
try:
cls.session.commit()
except:
cls.session.rollback()
raise
else:
obj = cls.create(**kwargs)
return obj
def update_or_create(cls, **kwargs):
"""Checks if an instance already exists by filtering with the
kwargs. If yes, updates the instance with new kwargs and
returns that instance. If not, creates a new
instance with kwargs and returns it.
Args:
**kwargs: The keyword arguments which are used for filtering
and initialization.
keys (list, optional): A special keyword argument. If passed,
only the set of keys mentioned here will be used for filtering.
Useful when we want to 'filter' based on a subset of the keys
and create with all the keys.
Examples:
>>> customer = Customer.update_or_create(
... name="vicky", email="*****@*****.**", country="India")
>>> customer.id
45
>>> customer1 = Customer.update_or_create(
... name="vicky", email="*****@*****.**", country="India")
>>> customer1==customer
True
>>> customer2 = Customer.update_or_create(
... name="vicky", email="*****@*****.**", country="Russia")
>>> customer2==customer
False
>>> customer3 = Customer.update_or_create(
... name="vicky", email="*****@*****.**", country="Russia",
... keys=['name', 'email'])
>>> customer3==customer
True
"""
keys = kwargs.pop('keys') if 'keys' in kwargs else []
filter_kwargs = subdict(kwargs, keys)
if filter_kwargs == {}:
obj = None
else:
obj = cls.first(**filter_kwargs)
if obj is not None:
for key, value in six.iteritems(kwargs):
if (key not in keys and key not in cls._no_overwrite_):
setattr(obj, key, value)
try:
cls.session.commit()
except:
cls.session.rollback()
raise
else:
obj = cls.create(**kwargs)
return obj
def update_or_build(cls, **kwargs):
keys = kwargs.pop('keys') if 'keys' in kwargs else []
filter_kwargs = subdict(kwargs, keys)
if filter_kwargs == {}:
obj = None
else:
obj = cls.first(**filter_kwargs)
if obj is not None:
for key, value in six.iteritems(kwargs):
if (key not in keys and key not in cls._no_overwrite_):
setattr(obj, key, value)
else:
obj = cls.build(**kwargs)
return obj
def update_or_build(cls, **kwargs):
keys = kwargs.pop('keys') if 'keys' in kwargs else []
filter_kwargs = subdict(kwargs, keys)
if filter_kwargs == {}:
obj = None
else:
obj = cls.first(**filter_kwargs)
if obj is not None:
for key, value in kwargs.iteritems():
if (key not in keys and
key not in cls._no_overwrite_):
setattr(obj, key, value)
else:
obj = cls.build(**kwargs)
return obj
def update_or_create(cls, **kwargs):
keys = kwargs.pop('keys') if 'keys' in kwargs else []
obj = cls.first(**subdict(kwargs, keys))
if obj is not None:
for key, value in kwargs.iteritems():
if (key not in keys and key not in cls.__no_overwrite__):
setattr(obj, key, value)
try:
cls.session.commit()
except:
cls.session.rollback()
raise
else:
obj = cls.create(**kwargs)
return obj
def find_or_create_all(cls, list_of_kwargs, keys=[]):
list_of_kwargs_wo_dupes, markers = remove_and_mark_duplicate_dicts(
list_of_kwargs, keys)
added_objs = cls.add_all([
cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
for kwargs in list_of_kwargs_wo_dupes
])
result_objs = []
iterator_of_added_objs = iter(added_objs)
for idx in range(len(list_of_kwargs)):
if idx in markers:
result_objs.append(added_objs[markers[idx]])
else:
result_objs.append(next(iterator_of_added_objs))
return result_objs
def update_or_create_all(cls, list_of_kwargs, keys=[]):
objs = []
for kwargs in list_of_kwargs:
obj = cls.first(**subdict(kwargs, keys))
if obj is not None:
for key, value in kwargs.iteritems():
if (key not in keys and key not in cls.__no_overwrite__):
setattr(obj, key, value)
else:
obj = cls.new(**kwargs)
objs.append(obj)
try:
return cls.add_all(objs)
except:
cls.session.rollback()
raise
def update_or_build_all(cls, list_of_kwargs, keys=[]):
"""Batch method for updating a list of instances and
creating them if required
Args:
list_of_kwargs(list of dicts): A list of dicts where
each dict denotes the keyword args that you would pass
to the create method separately
keys (list, optional): A list of keys to use for the
initial finding step. Matching is done only on these
attributes.
Examples:
>>> Customer.update_or_create_all([
... {'name': 'Vicky', 'email': '*****@*****.**', 'age': 34},
... {'name': 'Ron', 'age': 40, 'email': '*****@*****.**',
... 'gender': 'Male'}], keys=['name', 'email'])
"""
objs = []
for kwargs in list_of_kwargs:
filter_kwargs = subdict(kwargs, keys)
if filter_kwargs == {}:
obj = None
else:
obj = cls.first(**filter_kwargs)
if obj is not None:
for key, value in kwargs.iteritems():
if (key not in keys and
key not in cls._no_overwrite_):
setattr(obj, key, value)
else:
obj = cls.new(**kwargs)
objs.append(obj)
try:
return cls.add_all(objs, commit=False)
except:
cls.session.rollback()
raise
def update_or_build_all(cls, list_of_kwargs, keys=[]):
"""Batch method for updating a list of instances and
creating them if required
Args:
list_of_kwargs(list of dicts): A list of dicts where
each dict denotes the keyword args that you would pass
to the create method separately
keys (list, optional): A list of keys to use for the
initial finding step. Matching is done only on these
attributes.
Examples:
>>> Customer.update_or_create_all([
... {'name': 'Vicky', 'email': '*****@*****.**', 'age': 34},
... {'name': 'Ron', 'age': 40, 'email': '*****@*****.**',
... 'gender': 'Male'}], keys=['name', 'email'])
"""
objs = []
for kwargs in list_of_kwargs:
filter_kwargs = subdict(kwargs, keys)
if filter_kwargs == {}:
obj = None
else:
obj = cls.first(**filter_kwargs)
if obj is not None:
for key, value in six.iteritems(kwargs):
if (key not in keys and key not in cls._no_overwrite_):
setattr(obj, key, value)
else:
obj = cls.new(**kwargs)
objs.append(obj)
try:
return cls.add_all(objs, commit=False)
except:
cls.session.rollback()
raise
def get_updated_or_new_obj(cls, kwargs=None, filter_keys=None):
if filter_keys is None:
filter_keys = []
if kwargs is None:
kwargs = {}
filter_kwargs = subdict(kwargs, filter_keys)
if filter_kwargs == {}:
obj = None
else:
obj = cls.first(**filter_kwargs)
if obj is not None:
update_kwargs = {
k: v for k, v in kwargs.iteritems()
if k not in filter_keys and k not in cls._no_overwrite_}
obj.update_without_commit(**update_kwargs)
# for key, value in kwargs.iteritems():
# if (key not in filter_keys and
# key not in cls._no_overwrite_):
# setattr(obj, key, value)
else:
obj = cls.new(**kwargs)
return obj
def construct_query_modifiers(query_modifiers=None,
allow_modification_via_requests=True):
default_query_modifiers = {
"page": None,
"per_page": 20,
"limit": None,
"offset": None,
"order_by": None,
"sort": "asc",
}
if query_modifiers is None:
query_modifiers = {}
query_modifiers = subdict(query_modifiers, QUERY_MODIFIERS)
query_modifiers = merge(default_query_modifiers, query_modifiers)
if allow_modification_via_requests:
query_modifiers = merge(query_modifiers,
fetch_query_modifiers_from_request())
for k in ['page', 'per_page', 'limit', 'offset']:
if k in query_modifiers:
query_modifiers[k] = null_safe_type_cast(int,
query_modifiers.get(k))
return query_modifiers
def find_or_create(cls, **kwargs):
"""Checks if an instance already exists by filtering with the
kwargs. If yes, returns that instance. If not, creates a new
instance with kwargs and returns it
Args:
**kwargs: The keyword arguments which are used for filtering
and initialization.
keys(list, optional): A special keyword argument.
If passed, only the set of keys mentioned here will be used
for filtering. Useful when we want to 'find' based on a subset
of the keys and create with all the keys
Examples:
>>> customer = Customer.find_or_create(
... name="vicky", email="*****@*****.**", country="India")
>>> customer.id
45
>>> customer1 = Customer.find_or_create(
... name="vicky", email="*****@*****.**", country="India")
>>> customer1==customer
True
>>> customer2 = Customer.find_or_create(
... name="vicky", email="*****@*****.**", country="Russia")
>>> customer2==customer
False
>>> customer3 = Customer.find_or_create(
... name="vicky", email="*****@*****.**", country="Russia",
... keys=['name', 'email'])
>>> customer3==customer
True
"""
keys = kwargs.pop('keys') if 'keys' in kwargs else []
return cls.first(**subdict(kwargs, keys)) or cls.create(**kwargs)
def get_updated_or_new_obj(cls, kwargs=None, filter_keys=None):
if filter_keys is None:
filter_keys = []
if kwargs is None:
kwargs = {}
filter_kwargs = subdict(kwargs, filter_keys)
if filter_kwargs == {}:
obj = None
else:
obj = cls.first(**filter_kwargs)
if obj is not None:
update_kwargs = {
k: v
for k, v in six.iteritems(kwargs)
if k not in filter_keys and k not in cls._no_overwrite_
}
obj.update_without_commit(**update_kwargs)
# for key, value in kwargs.iteritems():
# if (key not in filter_keys and
# key not in cls._no_overwrite_):
# setattr(obj, key, value)
else:
obj = cls.new(**kwargs)
return obj
def find_or_create(cls, **kwargs):
"""Checks if an instance already exists by filtering with the
kwargs. If yes, returns that instance. If not, creates a new
instance with kwargs and returns it
Args:
**kwargs: The keyword arguments which are used for filtering
and initialization.
keys(list, optional): A special keyword argument.
If passed, only the set of keys mentioned here will be used
for filtering. Useful when we want to 'find' based on a subset
of the keys and create with all the keys
Examples:
>>> customer = Customer.find_or_create(
... name="vicky", email="*****@*****.**", country="India")
>>> customer.id
45
>>> customer1 = Customer.find_or_create(
... name="vicky", email="*****@*****.**", country="India")
>>> customer1==customer
True
>>> customer2 = Customer.find_or_create(
... name="vicky", email="*****@*****.**", country="Russia")
>>> customer2==customer
False
>>> customer3 = Customer.find_or_create(
... name="vicky", email="*****@*****.**", country="Russia",
... keys=['name', 'email'])
>>> customer3==customer
True
"""
keys = kwargs.pop('keys') if 'keys' in kwargs else []
return cls.first(**subdict(kwargs, keys)) or cls.create(**kwargs)
def find_or_new(cls, **kwargs):
keys = kwargs.pop('keys') if 'keys' in kwargs else []
return cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
([['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XL-INCH', 1],
['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-CDCHFLGOSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-NMSKU-SNACKDOWNBADGE', 1]],
[['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XL-INCH', 1],
['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-RFRECHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-NMSKU-SNACKDOWNBADGE', 1]]),
'XXL':
[['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XXL-INCH', 1],
['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-PRTECHFSTCKR-V14-ST-DCU-3X3D-C7FC-OP-3X3-INCH', 1],
['U326-NMSKU-SNACKDOWNBADGE', 1]],
'XXXL':
[['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XXXL-INCH', 1],
['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-PRTECHFSTCKR-V14-ST-DCU-3X3D-C7FC-OP-3X3-INCH', 1],
['U326-NMSKU-SNACKDOWNBADGE', 1]]
}
shipments = create_shipments(skus=skus,
recipients=recipients,
key=key,
secret=secret)
print "Read %s rows from the sheet" % sum(
len(v) for v in recipients.values())
print "Split up to send"
print "Found %s valid entries for shipments" % sum(
len(v) for v in subdict(recipients, skus.keys()).values())
print "Created following %s shipments:" % len(shipments)
print shipments
def fetch_query_modifiers_from_request():
return subdict(request.args, QUERY_MODIFIERS)
def find_or_new(cls, **kwargs):
keys = kwargs.pop('keys') if 'keys' in kwargs else []
return cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
[['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XL-INCH', 1],
['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-CDCHFLGOSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-NMSKU-SNACKDOWNBADGE', 1]],
[['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XL-INCH', 1],
['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-RFRECHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-NMSKU-SNACKDOWNBADGE', 1]]
),
'XXL': [['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XXL-INCH', 1],
['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-PRTECHFSTCKR-V14-ST-DCU-3X3D-C7FC-OP-3X3-INCH', 1],
['U326-NMSKU-SNACKDOWNBADGE', 1]],
'XXXL': [['U326-SNCKDWN-VSTIK-TS-RNE-PC-PLYCTT9AA-GRA-XXXL-INCH', 1],
['U326-SPRHROCHFSTCKR-V14-ST-DCU-2.5X2.B76-OP-2.5X2.5-INCH', 1],
['U326-PRTECHFSTCKR-V14-ST-DCU-3X3D-C7FC-OP-3X3-INCH', 1],
['U326-NMSKU-SNACKDOWNBADGE', 1]]
}
shipments = create_shipments(
skus=skus, recipients=recipients, key=key, secret=secret)
print "Read %s rows from the sheet" % sum(
len(v) for v in recipients.values())
print "Split up to send"
print "Found %s valid entries for shipments" % sum(
len(v) for v in subdict(recipients, skus.keys()).values())
print "Created following %s shipments:" % len(shipments)
print shipments
def find_or_new_all(cls, list_of_kwargs, keys=[]):
return [cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs) for kwargs in list_of_kwargs]
def find_or_new_all(cls, list_of_kwargs, keys=[]):
return [
cls.first(**subdict(kwargs, keys)) or cls.new(**kwargs)
for kwargs in list_of_kwargs
]