def validate_python(self, vals, state): program = vals.get('program') file = vals.get('data') guidelines_url = state.urls.build('general.guidelines', {}) if program and file: for d in convert_file(get_program(program), file): if isinstance(d, Exception): raise Invalid('', vals, state, error_dict={'data': 'There was an error processing the file. Please follow the \ <a href="javascript:popUp(\'' + guidelines_url + '\')">guidelines</a> [pop-up]'})
def addEntry(self, program, file, group): """ Calls the generator `convert_file()` found in :doc:`converters` on each row of the file, and adds the result to the database. If an exception happens during the converting and adding of data, then the session is rolled back and `None` is returned. Otherwise `True` is returned. .. note:: This had been optimised to make the adding of data as fast as possible, but has been slowed down again by adding search terms. **ToDo**: Optimise the adding of search terms. """ session.flush() browser_ids = {} try: entry_ins = Entry.__table__.insert() url_ins = URL.__table__.insert() count_num = 0 for d in convert_file(program, file): browser_name = d.pop('browser_name') browser_version = d.pop('browser_version') source = d.pop('source_file') key = (browser_name, browser_version, source) browser_id = browser_ids.get(key) if browser_id is None: browser = Browser.getFilterBy(name=browser_name, version=browser_version, source=source).first() if browser is None: browser = Browser(*key) session.add(browser) session.flush() browser_id = browser_ids[key] = browser.id # optimised to make adding data as fast as possible - ignores the ORM v = d.pop('access_time') if v is not None: d['access_date'] = datetime(v.year, v.month, v.day, 0, 0, 0, 0) d['access_time'] = time(v.hour, v.minute, v.second, v.microsecond) else: continue # don't add data without an access time v = d.pop('modified_time') if v is not None: d['modified_date'] = datetime(v.year, v.month, v.day, 0, 0, 0, 0) d['modified_time'] = time(v.hour, v.minute, v.second, v.microsecond) else: d['modified_date'] = None d['modified_time'] = None result = session.execute(entry_ins.values(browser_id=browser_id, group_id=group.id, **d)) entry_id = result.last_inserted_ids()[0] # add URLS url = URL(d['url']) session.execute(url_ins.values(entry_id=entry_id, **url.asDict())) url_id = result.last_inserted_ids()[0] # add search terms # TODO: make this optimised like above! entry = Entry.get(entry_id) url = URL.get(url_id) opts = config.options('search_engines') if url.query != None and 'search' in url.path: for opt in opts: if opt in url.netloc: query = url.query.split(config.get('search_engines', opt)+'=')\ [-1].split('&')[0] q_string, terms = SearchTerms.getTerms(urllib.unquote(query)) url.search = q_string for term in terms: t = SearchTerms.getFilterBy(term=term, engine=opt).first() if t is None: t = SearchTerms(term, opt, config.get('search', opt)) session.add(t) else: t.occurrence = t.occurrence + 1 entry.search_terms.append(t) session.flush() count_num = count_num + 1 if count_num == 0: # we have not added anything, but no exceptions where raised. return None, "No entries found in the uploaded file" except Exception, e: session.rollback() return None, e