def update_all(self, limit=None, time_limit=None):
"""Update All"""
logger.debug("Begin update_all(limit=%s)", limit)
transaction.begin()
self.ts = self.term_stat('SupplierCatalogItemField Update')
start_time = datetime.now()
try:
self.plugins = self.load_plugins()
query = DBSession.query(SupplierCatalogItemFieldModel)
if limit is not None:
query = query.order_by(SupplierCatalogItemFieldModel.updated.nullsfirst())
query = query.limit(limit)
self.ts['total'] = query.count()
for supplier_catalog_item_field in query.yield_per(10000):
self.update_one(supplier_catalog_item_field)
if self.ts['done'] % 10000 == 0:
DBSession.flush()
if time_limit is not None:
if datetime.now() > start_time + time_limit:
logger.info("Reached Time Limit at %i of %i", ts['done'], ts['total'])
break;
self.ts['done'] += 1
del query
transaction.commit()
except Exception:
logger.exception("Caught Exception: ")
transaction.abort()
finally:
self.ts.finish()
logger.debug("End update_all()")
def sort(self):
logger.debug("Begin sort()")
tx = transaction.get()
try:
query = DBSession.query(SupplierModel)
ts = self.term_stat('SupplierCatalog Sort', query.count())
suppliers = query.all()
for supplier in suppliers:
#print supplier
query = DBSession.query(SupplierCatalogModel)
query = query.filter(SupplierCatalogModel.supplier_id == supplier.id)
query = query.order_by(SupplierCatalogModel.issue_date)
prev_supplier_catalog = None
for supplier_catalog in query:
#print supplier_catalog
if prev_supplier_catalog is not None:
prev_supplier_catalog.next_supplier_catalog_id = supplier_catalog.id
supplier_catalog.prev_supplier_catalog_id = prev_supplier_catalog.id
supplier_catalog.next_supplier_catalog_id = None
prev_supplier_catalog = supplier_catalog
ts['sub_done'] += 1
ts['done'] += 1
ts['sub_done'] = 0
except Exception:
logger.exception("Caught Exception: ")
tx.rollback()
finally:
ts.finish()
logger.debug("End sort()")
def update_encoding(self):
logger.debug("Begin update_encoding()")
ts = self.term_stat('SupplierCatalog UpdateEncoding')
plugins = self.load_plugins()
tx = transaction.get()
try:
query = DBSession.query(SupplierCatalogModel)
query = query.order_by(desc(SupplierCatalogModel.created))
ts['total'] = query.count()
for supplier_catalog in query.yield_per(100):
if supplier_catalog.supplier_catalog_filter_id is None:
continue
#if supplier_catalog.encoding is None:
if True:
plug = plugins[supplier_catalog.supplier_catalog_filter_id]
encoding = plug.get_encoding(supplier_catalog)
if encoding is None:
encoding = plug.default_encoding
supplier_catalog.encoding = encoding['encoding']
DBSession.expunge(supplier_catalog.file_import)
ts['done'] += 1
except Exception:
logger.exception("Caught Exception: ")
tx.rollback()
finally:
ts.finish()
logger.debug("update_encoding()")
def load_all(self, modified_since=None):
"""Load All"""
logger.debug("Begin load_all()")
ts = self.term_stat('SupplierCatalog Load')
tx = transaction.get()
try:
plugins = self.load_plugins()
query = DBSession.query(FileImportModel)
if modified_since:
query = query.filter(FileImportModel.modified >= modified_since)
ts['total'] = query.count()
for file_import in query.yield_per(1):
#print file_import.name
for plug in plugins.itervalues():
is_match = plug.match_file_import(file_import)
if is_match:
self.load_one(plug, file_import)
break
DBSession.expunge(file_import)
ts['done'] += 1
except Exception:
logger.exception("Caught Exception: ")
tx.abort()
finally:
ts.finish()
transaction.commit()
logger.debug("End load_all()")
def update_all(self, limit=None, time_limit=None):
"""Update All"""
logger.debug("Begin update_all()")
ts = self.term_stat('Products Update')
start_time = datetime.now()
tx = transaction.get()
try:
query = DBSession.query(ProductModel)
if limit is not None:
query = query.order_by(ProductModel.updated.nullsfirst())
query = query.limit(limit)
ts['total'] = query.count()
for product in query.yield_per(100):
self.update_one(product)
if ts['done'] % 100 == 0:
DBSession.flush()
if time_limit is not None:
if datetime.now() > start_time + time_limit:
logger.info("Reached Time Limit at %i of %i", ts['done'], ts['total'])
break;
ts['done'] += 1
except Exception:
logger.exception("Caught Exception: ")
tx.abort()
finally:
ts.finish()
logger.debug("End update_all()")
def update_customer_order_items(self, product): """Update Customer Order Items""" query = DBSession.query(CustomerOrderItemModel) query = query.filter(CustomerOrderItemModel.product_id == product.id) product.customer_order_item_count = query.count() for customer_order_item in query: query2 = DBSession.query(CustomerShipmentItemModel) query2 = query.filter(CustomerShipmentItemModel.customer_order_item_id == customer_order_item.id) product.customer_shipment_item_count = query2.count()
def vacuum_all(self, limit=None, time_limit=None, unupdated=False):
logger.debug('Begin vacuum_all(limit=%s, time_limit=%s, unupdated=%s)', limit, time_limit, unupdated)
##TODO delete SCIFields with SCFilterId not found in SCFilter
self.plugins = self.load_plugins()
self.ts = self.term_stat('SupplierCatalogItemFields Vacuum', len(self.plugins))
now = start_time = datetime.now()
try:
transaction.begin()
for plug in self.plugins.itervalues():
supplier_catalog_filter_id = plug.supplier_catalog_filter_id()
### Generate a bloom filter set of SCIF id's in VersionModel
model_name = plug.version_model() + 'Model'
VersionModel = getattr(model, model_name)
query = DBSession.query(VersionModel.supplier_catalog_item_field_id)
s = BloomFilter(capacity=query.count() + 1)
self.ts['sub_total'] = query.count()
for (supplier_catalog_item_field_id, ) in query.yield_per(100):
s.add(supplier_catalog_item_field_id)
self.ts['sub_done'] += 1
del query
### Iterate through SCIFields, deleting any that don't appear in the bloom filter.
query = DBSession.query(SupplierCatalogItemFieldModel)
query = query.filter(SupplierCatalogItemFieldModel.supplier_catalog_filter_id == supplier_catalog_filter_id)
if unupdated is not True:
query = query.filter(SupplierCatalogItemFieldModel.updated != None)
if limit is not None:
query = query.order_by(SupplierCatalogItemFieldModel.vacuumed.nullsfirst())
query = query.limit(limit)
logger.debug("LIMIT %i, supplier_catalog_filter_id %s", limit, supplier_catalog_filter_id)
self.ts['sub_done'] = 0
self.ts['sub_total'] = query.count()
for supplier_catalog_item_field in query.yield_per(100):
if supplier_catalog_item_field.id not in s:
logger.debug("Deleting SupplierCatalogItemField %s", supplier_catalog_item_field.id)
DBSession.delete(supplier_catalog_item_field)
else:
supplier_catalog_item_field.vacuumed = now
if self.ts['sub_done'] % 1000 == 0:
DBSession.flush()
self.ts['sub_done'] += 1
del query
DBSession.flush()
if time_limit is not None:
if datetime.now() > start_time + time_limit:
logger.info("Reached Time Limit at %i of %i", self.ts['done'], self.ts['total'])
transaction.commit()
break;
self.ts['done'] += 1
transaction.commit()
except Exception:
logger.exception("Caught Exception: ")
transaction.abort()
finally:
self.ts.finish()
logger.debug('End vacuum()')
def load_one(self, supplier_catalog_item): """Load One""" query = DBSession.query(ProductModel) query = query.filter(ProductModel.manufacturer_id == supplier_catalog_item.manufacturer_id) query = query.filter(ProductModel.identifier == supplier_catalog_item.product_identifier) count = query.count() if (count == 0): product = ProductModel() product.manufacturer_id = supplier_catalog_item.manufacturer_id product.identifier = supplier_catalog_item.product_identifier DBSession.add(product) supplier_catalog_item_task = SupplierCatalogItemTask() supplier_catalog_item_task.update_product(supplier_catalog_item)
def load_all(self):
"""Load All"""
logger.debug("Begin load_all()")
tx = transaction.get()
try:
query = DBSession.query(SupplierCatalogItemModel)
query = query.filter(SupplierCatalogItemModel.manufacturer_identifier != None)
query = query.filter(SupplierCatalogItemModel.manufacturer_id != None)
query = query.filter(SupplierCatalogItemModel.product_identifier != None)
query = query.filter(SupplierCatalogItemModel.product_id == None)
query = query.filter(SupplierCatalogItemModel.category_id != None)
query = query.filter(SupplierCatalogItemModel.scale_id != None)
query = query.filter(SupplierCatalogItemModel.phased_out == False)
query = query.filter(
or_(
SupplierCatalogItemModel.in_stock == True,
SupplierCatalogItemModel.advanced == True
)
)
ts = self.term_stat('Products Load', query.count())
for supplier_catalog_item in query.yield_per(1000):
self.load_one(supplier_catalog_item)
ts['done'] += 1
except Exception:
logger.exception("Caught Exception: ")
tx.abort()
finally:
ts.finish()
logger.debug("End load_all()")
def load_one(self, plug, file_import): """Load One""" query = DBSession.query(SupplierCatalogModel) query = query.filter(SupplierCatalogModel.file_import_id == file_import.id) if query.count() == 0: supplier_catalog = SupplierCatalogModel() DBSession.add(supplier_catalog) supplier_catalog.file_import_id = file_import.id else: supplier_catalog = query.one() supplier_catalog.supplier_id = plug.supplier_id() supplier_catalog.supplier_catalog_filter_id = plug.supplier_catalog_filter_id() if not supplier_catalog.lock_issue_date: supplier_catalog.issue_date = plug.issue_date(file_import)
def sort(self):
try:
logger.info("Caching Manufacturers...")
manufacturers = dict()
query = DBSession.query(ManufacturerModel)
for manufacturer in query:
manufacturers[manufacturer.id] = manufacturer.name
query = DBSession.query(ProductModel)
query = query.order_by(ProductModel.sort)
sorttable = list()
logger.info("Generating List...")
for product in query:
data = [
product.id,
manufacturers[product.manufacturer_id],
product.identifier
]
sorttable.append(data)
def sortkey(s):
return (alphanum_key(s[1]), alphanum_key(s[2]))
logger.info("Sorting List...")
sorttable.sort(key=sortkey)
ts = self.term_stat('Products Sort', len(sorttable))
for x in xrange(len(sorttable)):
(product_id, a, b) = sorttable[x]
query = DBSession.query(ProductModel)
#query = query.filter(ProductModel.id == product_id)
#product = query.one()
product = query.get(product_id)
product.sort = x
ts['done'] += 1
except Exception:
logger.exception("Caught Exception: ")
transaction.abort()
finally:
ts.finish()
def get_supplier_catalog_item(self, product): """Get Supplier Catalog Item""" #print "Product", product.id data = dict() data['phased_out'] = False data['in_stock'] = False data['advanced'] = False data['supplier_catalog_item'] = None data['category_id'] = None data['scale_id'] = None query = DBSession.query(SupplierCatalogItemModel) query = query.filter(SupplierCatalogItemModel.product_id == product.id) query = query.filter(SupplierCatalogItemModel.rank > 0) query.order_by(asc(SupplierCatalogItemModel.rank)) count = data['supplier_catalog_item_count'] = query.count() if count == 0: return data supplier_catalog_items = query.all() #*** Find SCI with highest rank having product in stock for supplier_catalog_item in supplier_catalog_items: #print "SCI", supplier_catalog_item.id if supplier_catalog_item.phased_out is True: data['phased_out'] = True if supplier_catalog_item.in_stock is True: data['in_stock'] = True data['supplier_catalog_item'] = supplier_catalog_item if supplier_catalog_item.category_id is not None: data['category_id'] = supplier_catalog_item.category_id if supplier_catalog_item.scale_id is not None: data['scale_id'] = supplier_catalog_item.scale_id if data['in_stock']: return data #*** Find SCI with highest rank having product on pre-order for supplier_catalog_item in supplier_catalog_items: #print "SCI", supplier_catalog_item.id if supplier_catalog_item.advanced is True: data['advanced'] = True data['supplier_catalog_item'] = supplier_catalog_item if data['advanced']: return data #*** since no supplier has stock, or advanced orders #*** we return the SCI with the highest rank data['supplier_catalog_item'] = supplier_catalog_items[-1] return data
def update_inventory_items(self, product): """Update Inventory Items""" query = DBSession.query(InventoryItemModel) query = query.filter(InventoryItemModel.product_id == product.id) product.inventory_item_count = query.count() quantity = Decimal(0) for inventory_item in query: quantity += inventory_item.quantity product.stock = quantity
def load_all(self, limit=None, item_versions_loaded=None, supplier_id=None):
"""Load All"""
logger.debug("Begin load_all(limit=%s, item_versions_loaded=%s)", limit, item_versions_loaded)
self.ts = self.term_stat('SupplierSpecialItemVersion Load')
tx = transaction.get()
try:
self.plugins = self.load_plugins()
query = DBSession.query(SupplierSpecialModel)
alt_query = query.filter(SupplierSpecialModel.supplier_special_item_versions_loaded == None)
if alt_query.count() > 0:
query = alt_query.order_by(desc(SupplierSpecialModel.begin_date))
else:
query = query.order_by(SupplierSpecialModel.supplier_special_item_versions_loaded.nullsfirst())
del alt_query
if supplier_id is not None:
query = query.filter(SupplierSpecialModel.supplier_id == supplier_id)
else:
query = query.filter(SupplierSpecialModel.supplier_id != None)
if limit is not None:
query = query.limit(limit)
self.ts['total'] = query.count()
for supplier_special in query.yield_per(10):
self.load_one(supplier_special)
supplier_special.supplier_special_item_versions_loaded = datetime.now()
if self.ts['done'] % 1000 == 0 :
DBSession.flush()
self.ts['done'] += 1
except Exception:
logger.exception('Caught Exception: ')
tx.abort()
finally:
self.ts.finish()
transaction.commit()
logger.debug("End load_all()")
def load_one(self, plug, file_import): """Load One""" query = DBSession.query(SupplierSpecialModel) query = query.filter(SupplierSpecialModel.file_import_id == file_import.id) if query.count() == 0: supplier_special = SupplierSpecialModel() DBSession.add(supplier_special) supplier_special.file_import_id = file_import.id else: supplier_special = query.one() supplier_special.supplier_id = plug.supplier_id() supplier_special.supplier_special_filter_id = plug.supplier_special_filter_id() #if not supplier_special.lock_issue_date: if True: dates = plug.issue_dates(file_import) if dates is not None: supplier_special.begin_date = dates[0] supplier_special.end_date = dates[1] print 'BEGIN', supplier_special.begin_date print 'END', supplier_special.end_date
def update_params(self, d): rows = DBSession.query(ScaleModel) #rows = rows.filter(ScaleModel.display == True) rows = rows.order_by(ScaleModel.name) options = [(None, '-')] for row in rows: options.append( ( row.id, row.name, ) ) d['options'] = options SingleSelectField.update_params(self, d) return d
def load_supplier_special_item_field(self, supplier_special, row):
if row is not None:
j = SupplierSpecialItemFieldModel.encode_json(row)
else:
j = None
if j is None:
supplier_special_item_field = None
else:
checksum = hashlib.sha1(j).hexdigest()
plug = self.plugins[supplier_special.supplier_special_filter_id]
query = DBSession.query(SupplierSpecialItemFieldModel)
query = query.filter(SupplierSpecialItemFieldModel.checksum == checksum)
try:
supplier_special_item_field = query.one()
except NoResultFound:
supplier_special_item_field = SupplierSpecialItemFieldModel()
DBSession.add(supplier_special_item_field)
supplier_special_item_field.fields = j.encode('utf-8')
supplier_special_item_field.checksum = checksum
supplier_special_item_field.supplier_id = supplier_special.supplier_id
supplier_special_item_field.supplier_special_filter_id = plug.supplier_special_filter_id()
return supplier_special_item_field
def load_supplier_special_item_version(self, supplier_special, supplier_special_item_field, row_number): plug = self.plugins[supplier_special.supplier_special_filter_id] model_name = plug.version_model() + 'Model' VersionModel = getattr(model, model_name) query = DBSession.query(VersionModel) query = query.filter(VersionModel.supplier_special_id == supplier_special.id) query = query.filter(VersionModel.row_number == row_number) if supplier_special_item_field is None: query.delete() supplier_special_item_version = None else: try: supplier_special_item_version = query.one() except NoResultFound: supplier_special_item_version = VersionModel() DBSession.add(supplier_special_item_version) supplier_special_item_version.supplier_special_id = supplier_special.id supplier_special_item_version.supplier_special_item_field_id = supplier_special_item_field.id supplier_special_item_version.supplier_special_filter_id = plug.supplier_special_filter_id() supplier_special_item_version.row_number = row_number #supplier_special_item_version.effective = supplier_special.issue_date return supplier_special_item_version
def setup(self):
try:
new_attrs = {}
new_attrs.update(self.attrs)
new_attrs.update(self.do_get_dependencies())
self.obj = self.klass(**new_attrs)
DBSession.add(self.obj)
DBSession.flush()
return self.obj
except:
DBSession.rollback()
raise
def update_params(self, d):
rows = DBSession.query(ManufacturerModel)
rows = rows.filter(ManufacturerModel.display == True)
rows = rows.order_by(ManufacturerModel.sort)
options = [(None, '-')]
for row in rows:
options.append(
(
row.id,
"{n:.32} - ({i})".format(
n=row.name,
i=row.identifier
)
)
)
d['options'] = options
SingleSelectField.update_params(self, d)
return d
def vacuum_all(self, limit=None):
logger.debug('Begin vacuum_all(limit=%s)', limit)
self.plugins = self.load_plugins()
ts = self.term_stat('SupplierSpecialItemVersion Vacuum', len(self.plugins))
tx = transaction.get()
try:
#s = set()
s = ScalableBloomFilter()
query = DBSession.query(SupplierSpecialModel.id)
for (supplier_special_id, ) in query.yield_per(100):
s.add(supplier_special_id)
for plug in self.plugins.itervalues():
supplier_special_filter_id = plug.supplier_special_filter_id()
model_name = plug.version_model() + 'Model'
VersionModel = getattr(model, model_name)
query = DBSession.query(VersionModel)
if limit:
query = query.order_by(VersionModel.vacuumed.nullsfirst())
query = query.limit(limit)
ts['sub_done'] = 0
ts['sub_total'] = query.count()
for supplier_special_item_version in query.yield_per(10):
if supplier_special_item_version.supplier_special_id not in s:
logger.debug("Deleting %s %s", model_name, supplier_special_item_version.id)
DBSession.delete(supplier_special_item_version)
ts['sub_done'] += 1
if ts['sub_done'] % 1000 == 0:
DBSession.flush()
DBSession.flush()
ts['done'] += 1
except Exception:
logger.exception('Caught Exception: ')
tx.abort()
finally:
ts.finish()
transaction.commit()
logger.debug('End vacuum_all()')
def by_email_address(cls, email):
"""Return the user object whose email address is ``email``."""
return DBSession.query(cls).filter(cls.email_address==email).first()
def test_query_obj(self):
obj = DBSession.query(self.klass).one()
for key, value in self.attrs.iteritems():
assert_equals(getattr(obj, key), value)
def tearDown(self):
DBSession.rollback()
def by_user_name(cls, username):
"""Return the user object whose user name is ``username``."""
return DBSession.query(cls).filter(cls.user_name==username).first()
