def load_all(self, modified_since=None):
"""Load All"""
logger.debug("Begin load_all()")
ts = self.term_stat('SupplierCatalog Load')
tx = transaction.get()
try:
plugins = self.load_plugins()
query = DBSession.query(FileImportModel)
if modified_since:
query = query.filter(FileImportModel.modified >= modified_since)
ts['total'] = query.count()
for file_import in query.yield_per(1):
#print file_import.name
for plug in plugins.itervalues():
is_match = plug.match_file_import(file_import)
if is_match:
self.load_one(plug, file_import)
break
DBSession.expunge(file_import)
ts['done'] += 1
except Exception:
logger.exception("Caught Exception: ")
tx.abort()
finally:
ts.finish()
transaction.commit()
logger.debug("End load_all()")
def update_encoding(self):
logger.debug("Begin update_encoding()")
ts = self.term_stat('SupplierCatalog UpdateEncoding')
plugins = self.load_plugins()
tx = transaction.get()
try:
query = DBSession.query(SupplierCatalogModel)
query = query.order_by(desc(SupplierCatalogModel.created))
ts['total'] = query.count()
for supplier_catalog in query.yield_per(100):
if supplier_catalog.supplier_catalog_filter_id is None:
continue
#if supplier_catalog.encoding is None:
if True:
plug = plugins[supplier_catalog.supplier_catalog_filter_id]
encoding = plug.get_encoding(supplier_catalog)
if encoding is None:
encoding = plug.default_encoding
supplier_catalog.encoding = encoding['encoding']
DBSession.expunge(supplier_catalog.file_import)
ts['done'] += 1
except Exception:
logger.exception("Caught Exception: ")
tx.rollback()
finally:
ts.finish()
logger.debug("update_encoding()")
