def filter_del(request):
""" UserFilter del view """
schema = UserFilterDelSchema()
schema = schema.bind(request=request)
form = SubmitForm(schema)
success_message = isinstance(request.context, RootFactory) \
and _(u'The filter has been removed')
if request.method == 'GET':
return dict(form=form,
appstruct=request.context.__json__())
else:
try:
controls = request.POST.items()
captured = form.validate(controls)
except deform.ValidationFailure, e:
return dict(form=e, errors=e.error.children)
filter = request.context
DBSession.delete(filter)
request.session.flash(success_message, 'success')
return HTTPFound(
location=request.route_url('filter_list')
)
def filter_del(request):
""" UserFilter del view """
schema = UserFilterDelSchema()
schema = schema.bind(request=request)
form = SubmitForm(schema)
success_message = isinstance(request.context, RootFactory) \
and _(u'The filter has been removed')
if request.method == 'GET':
return dict(form=form, appstruct=request.context.__json__())
else:
try:
controls = request.POST.items()
captured = form.validate(controls)
except deform.ValidationFailure, e:
return dict(form=e, errors=e.error.children)
filter = request.context
DBSession.delete(filter)
request.session.flash(success_message, 'success')
return HTTPFound(location=request.route_url('filter_list'))
def getLastScrappedDocId():
lastId = DBSession.query(Setting.value).filter(Setting.key == u'lastFactorizedDocId').all()
if len(lastId) == 0:
DBSession.add(Setting(key = u'lastFactorizedDocId', value = u'0'))
transaction.commit()
return 0
else:
return int(lastId[0][0])
def load_config(filename="development.ini", basePath="."):
config = load_settings(filename, basePath)
engine = engine_from_config(config, "sqlalchemy.")
DBSession.configure(bind=engine)
initialize_sql(engine)
def load_config(filename='development.ini', basePath='.'):
config = load_settings(filename, basePath)
engine = engine_from_config(config, 'sqlalchemy.')
DBSession.configure(bind=engine)
initialize_sql(engine)
def saveFactsToDatabase(docid, facts, intFacts):
data = FactData()
data.scraped_data_id = docid
for fact in facts:
setattr(data, fact, True)
for name, value in intFacts:
setattr(data, name, value)
DBSession.add(data)
transaction.commit()
def getLastScrappedDocId():
lastId = DBSession.query(
Setting.value).filter(Setting.key == u'lastFactorizedDocId').all()
if len(lastId) == 0:
DBSession.add(Setting(key=u'lastFactorizedDocId', value=u'0'))
transaction.commit()
return 0
else:
return int(lastId[0][0])
def saveFactsToDatabase(docid, facts, intFacts):
data = FactData()
data.scraped_data_id = docid
for fact in facts:
setattr(data, fact, True)
for name, value in intFacts:
setattr(data, name, value)
DBSession.add(data)
transaction.commit()
def setUp(self):
connection = self.engine.connect()
# begin a non-ORM transaction
self.trans = connection.begin()
# bind an individual Session to the connection
DBSession.configure(bind=connection)
self.session = self.Session(bind=connection)
Base.session = self.session
def setUp(self):
connection = self.engine.connect()
# begin a non-ORM transaction
self.trans = connection.begin()
# bind an individual Session to the connection
DBSession.configure(bind=connection)
self.session = self.Session(bind=connection)
Base.session = self.session
def uploadFacts():
import sys
from factCodeBase import facts, factLinks
initializeBase(sys.argv[1], sys.argv[2])
for key, value in facts.items():
DBSession.add(Fact(name=unicode(key), type=FACT_TYPE_BOOL, args=unicode(json.encode(value))))
for key, value in factLinks.items():
DBSession.add(FactLinks(name=unicode(key), args=json.encode(value)))
transaction.commit()
def uploadFacts():
import sys
from factCodeBase import facts, factLinks
initializeBase(sys.argv[1], sys.argv[2])
for key, value in facts.items():
DBSession.add(
Fact(name=unicode(key),
type=FACT_TYPE_BOOL,
args=unicode(json.encode(value))))
for key, value in factLinks.items():
DBSession.add(FactLinks(name=unicode(key), args=json.encode(value)))
transaction.commit()
def filter_add(request):
""" UserFilter add/edit view """
schema = UserFilterSchema()
schema = schema.bind(request=request)
form = SubmitForm(schema)
form.children[-1].renderer = renderer
success_message = isinstance(request.context, RootFactory) \
and _(u'The filter was created') \
or _(u'The filter was updated')
if request.method == 'GET':
if isinstance(request.context, RootFactory):
return dict(form=form)
else:
return dict(form=form,
appstruct=request.context.__json__())
else:
try:
controls = request.POST.items()
captured = form.validate(controls)
except deform.ValidationFailure, e:
return dict(form=e, errors=e.error.children)
if isinstance(request.context, RootFactory):
filter = UserFilter(user=request.user, name=captured['name'], filter_details=captured['filter_details'])
else:
filter = request.context
filter.name = captured['name']
filter.filter_details = captured['filter_details']
DBSession.add(filter)
request.session.flash(success_message, 'success')
return HTTPFound(
location=request.route_url('filter_list')
)
def filter_add(request):
""" UserFilter add/edit view """
schema = UserFilterSchema()
schema = schema.bind(request=request)
form = SubmitForm(schema)
form.children[-1].renderer = renderer
success_message = isinstance(request.context, RootFactory) \
and _(u'The filter was created') \
or _(u'The filter was updated')
if request.method == 'GET':
if isinstance(request.context, RootFactory):
return dict(form=form)
else:
return dict(form=form, appstruct=request.context.__json__())
else:
try:
controls = request.POST.items()
captured = form.validate(controls)
except deform.ValidationFailure, e:
return dict(form=e, errors=e.error.children)
if isinstance(request.context, RootFactory):
filter = UserFilter(user=request.user,
name=captured['name'],
filter_details=captured['filter_details'])
else:
filter = request.context
filter.name = captured['name']
filter.filter_details = captured['filter_details']
DBSession.add(filter)
request.session.flash(success_message, 'success')
return HTTPFound(location=request.route_url('filter_list'))
def parse():
lastId = getLastScrappedDocId()
factStorage = createFactStorage()
backFacts = backFactLinksProcessing()
newLastId = lastId
for newLastId, url, rawText, title, data in DBSession.query(
ScrapedData.pk, ScrapedData.url, ScrapedData.preview,
ScrapedData.title,
ScrapedData.data).filter(ScrapedData.pk > lastId).order_by(
ScrapedData.pk): #.limit(10):
facts = extractAll(rawText, factStorage, backFacts)
facts.update(extractAll(title, factStorage, backFacts))
#print url, facts
data = json.decode(data)
salaryFact = []
if 'salary' in data:
salary = extractSalary(data['salary'])
if salary is not None:
if len(salary) == 2:
try:
salaryCount = int(re.sub('\s*', '', salary[0], 0,
re.U))
except ValueError, x:
salaryCount = 0
if salary[1].strip() == '$' or \
salary[1].strip().lower() == 'usd':
salaryCount *= 8
salaryFact.append(('salary_from', salaryCount))
elif len(salary) == 3:
try:
salaryFrom = int(re.sub('\s*', '', salary[0], 0, re.U))
salaryTo = int(re.sub('\s*', '', salary[1], 0, re.U))
except ValueError:
salaryFrom = 0
salaryTo = 0
if salary[2].strip() == '$' or\
salary[2].strip().lower() == 'usd':
salaryFrom *= 8
salaryTo *= 8
salaryFact.append(('salary_from', salaryFrom))
salaryFact.append(('salary_to', salaryTo))
saveFactsToDatabase(newLastId, facts, salaryFact)
def parse():
lastId = getLastScrappedDocId()
factStorage = createFactStorage()
backFacts = backFactLinksProcessing()
newLastId = lastId
for newLastId, url, rawText, title, data in DBSession.query(ScrapedData.pk, ScrapedData.url, ScrapedData.preview, ScrapedData.title, ScrapedData.data).filter(ScrapedData.pk > lastId).order_by(ScrapedData.pk):#.limit(10):
facts = extractAll(rawText, factStorage, backFacts)
facts.update(extractAll(title, factStorage, backFacts))
#print url, facts
data = json.decode(data)
salaryFact = []
if 'salary' in data:
salary = extractSalary(data['salary'])
if salary is not None:
if len(salary) == 2:
try:
salaryCount = int(re.sub('\s*', '', salary[0], 0, re.U))
except ValueError, x:
salaryCount = 0
if salary[1].strip() == '$' or \
salary[1].strip().lower() == 'usd':
salaryCount *= 8
salaryFact.append(('salary_from', salaryCount))
elif len(salary) == 3:
try:
salaryFrom = int(re.sub('\s*', '', salary[0], 0, re.U))
salaryTo = int(re.sub('\s*', '', salary[1], 0, re.U))
except ValueError:
salaryFrom = 0
salaryTo = 0
if salary[2].strip() == '$' or\
salary[2].strip().lower() == 'usd':
salaryFrom *= 8
salaryTo *= 8
salaryFact.append(('salary_from', salaryFrom))
salaryFact.append(('salary_to', salaryTo))
saveFactsToDatabase(newLastId, facts, salaryFact)
def loadFactsFromDatabase():
return DBSession.query(Fact).all()
def loadFactsFromDatabase():
return DBSession.query(Fact).all()
def loadFactLinksFromDatabase():
return DBSession.query(FactLinks).filter(Fact.type == FACT_TYPE_BOOL)
def putLastScrappedDocId(lastId):
DBSession.query(
Setting.value).filter(Setting.key == u'lastFactorizedDocId').update(
{u'value': unicode(lastId)})
transaction.commit()
def putLastScrappedDocId(lastId):
DBSession.query(Setting.value).filter(Setting.key == u'lastFactorizedDocId').update({u'value': unicode(lastId)})
transaction.commit()
def loadFactLinksFromDatabase():
return DBSession.query(FactLinks).filter(Fact.type == FACT_TYPE_BOOL)