def connect(self, log=False): """ dialect+driver://username:password@host:port/database """ if self.driver: connect_str = '{0}+{1}://{2}:{3}@{4}/{5}'.format( self.dialect, self.driver, self.user, self.pwrd, self.host, self.db, ) else: connect_str = '{0}://{1}:{2}@{3}/{4}'.format( self.dialect, self.user, self.pwrd, self.host, self.db, ) if log: log.info('Establishing connection to "%s://%s@%s/%s"' % ( self.dialect, self.user, self.host, self.db )) self.engine = create_engine(connect_str) self.make_session()
def manage_article_storage(max_article_id, article_count): """ Handle the storage of new articles :param max_article_id: int; article id :param article_count: int; total count of articles :return: bool; success """ if article_count >= settings.MYSQL_MAX_ROWS: if max_article_id: # TODO - CHANGE THIS be careful, could iterate many times article_removed = False attempts = 0 while not article_removed \ or attempts > settings.MAX_RETRIES_FOR_REMOVE: attempts += 1 article_id = random.randint(0, int(max_article_id)) with ArticleModel() as am: log.info('Removing article id: ' + str(article_id)) try: am.delete_article(article_id) article_removed = True except UnmappedInstanceError: continue else: log.error('Could not determine a max article id.') return True
def login(): if request.method == 'POST' and 'username' in request.form: username = escape(str(request.form['username'].strip())) passwd = escape(str(request.form['password'].strip())) remember = request.form.get('remember', 'no') == 'yes' log.info('Attempting login for "%s"' % username) with UserModel() as um: user = um.fetch_user_by_name(username) if not user: log.info('On login - User not found "%s"' % username) flash('Login failed.') return render_template('login.html') user_ref = User(user._id) user_ref.authenticate(passwd) if user_ref.is_authenticated(): login_user(user_ref, remember=remember) flash('Logged in.') return redirect(request.args.get('next') or url_for('home')) else: flash('Login failed.') return render_template('login.html')
def fetch_user_by_name(self, handle): query_obj = self.io.session.query(schema.User).filter( schema.User.handle == handle) try: return self.alchemy_fetch_validate(query_obj, RET_TYPE_FIRSTROW) except (KeyError, IndexError) as e: log.info('User not found "%s": %s' % (handle, e.message)) return None
def get_serialized(type, key): """Retrieves the obj from pickle file for this type/key :param type: object type to pickle :param key: unique id :return: serialized object """ fname = getPicklerFilename(type, key) with open(fname, 'rb') as f: log.info('Fetching pickle: ' + fname) return cPickle.load(f)
def set_serialized(obj, type, key): """Sets the pickle file for this type/key :param obj: object to serialize :param type: object type to pickle :param key: unique id """ fname = getPicklerFilename(type, key) with open(fname, 'wb') as f: log.info('Setting pickle: ' + fname) cPickle.dump(obj, f)
def get_upload(self, flickr_photo_id): """ Retrieve a photo by its flickr id """ log.info('Fetching photo by flickr_photo_id: %s' % flickr_photo_id) schema_obj = getattr(schema, 'Upload') query_obj = self.io.session.query(schema_obj).filter( schema_obj.flickr_photo_id == flickr_photo_id) res = self.alchemy_fetch_validate(query_obj) if len(res) > 0: return res[0] else: return None
def init_views(): for key in view_list: # Add maintenance wrapper # view_list[key] = maintenance(view_list[key]) # wrap methods for login requirement if key not in views_with_anonymous_access: view_list[key] = login_required(view_list[key]) for key in route_deco: route = route_deco[key] view_method = view_list[key] view_list[key] = route(view_method) log.info(__name__ + ' :: Registered views - {0}'.format(str(view_list)))
def get_article_stored_body(article): """ Fetch corresponding article object :param article: str; article name :return: json, Article; stored page content, corresponding article model object """ with ArticleModel() as am: article_obj = am.get_article_by_name(article) try: with ArticleContentModel() as acm: body = acm.get_article_content(article_obj._id).markup except Exception as e: log.info('Article markup not found: "%s"' % e.message) body = '' return body
def handle_photo_integrate(photos, html, article): """ Integrate photo link tags into content. This walks through each section header and inserts an image below the header. :param photos: List of photo meta-info :param html: Wiki-html :return: modified content """ soup = BeautifulSoup(html) photo_index = 0 # Embed Title photo lf = '<div style="clear:both;"> </div>' try: tag = embed_photo_content( article, photo_index, photos[photo_index], soup, TITLE_PHOTO_SIZE_X, TITLE_PHOTO_SIZE_Y ) except (ValueError, KeyError, IndexError): log.info('In parse no photos found') return html html = str(tag) + lf + lf + html photo_index += 1 # Embed section photos headers = soup.findAll('h2') headers.extend(soup.findAll('h3')) for node in headers: if len(photos) > photo_index: tag = embed_photo_content( article, photo_index, photos[photo_index], soup ) html = html.replace(str(node), str(node) + str(tag)) photo_index += 1 else: break return html
def api(method): """REST interface for flickipedia - swtches on method calls""" # Extract photo-id, article-id, user-id article_id = request.args.get('article-id') user_id = request.args.get('user-id') photo_id = request.args.get('photo-id') if method == API_METHOD_ENDORSE_EVENT: log.info('On %s getting (article, user, photo) = (%s, %s, %s)' % ( API_METHOD_ENDORSE_EVENT, article_id, user_id, photo_id)) api_method_endorse_event(article_id, user_id, photo_id) return Response(json.dumps(['endorse-event']), mimetype='application/json') elif method == API_METHOD_ENDORSE_FETCH: log.info('On %s getting (article, user, photo) = (%s, %s, %s)' % ( API_METHOD_ENDORSE_FETCH, article_id, user_id, photo_id)) res = api_method_endorse_fetch(article_id, user_id, photo_id) return Response(json.dumps({'endorse-fetch': res}), mimetype='application/json') elif method == API_METHOD_EXCLUDE_EVENT: api_method_exclude_event(article_id, user_id, photo_id) return Response(json.dumps(['exclude-event']), mimetype='application/json') elif method == API_METHOD_EXCLUDE_FETCH: log.info('On %s getting (article, user, photo) = (%s, %s, %s)' % ( API_METHOD_EXCLUDE_FETCH, article_id, user_id, photo_id)) res = api_method_exclude_fetch(article_id, user_id, photo_id) return Response(json.dumps({'exclude-fetch': res}), mimetype='application/json') elif method == API_METHOD_ENDORSE_COUNT: log.info('On %s getting (article, photo) = (%s, %s)' % ( API_METHOD_EXCLUDE_COUNT, article_id, photo_id)) res = api_method_endorse_count(article_id, photo_id) return Response(json.dumps({'endorse-count': res}), mimetype='application/json') elif method == API_METHOD_EXCLUDE_COUNT: log.info('On %s getting (article, photo) = (%s, %s)' % ( API_METHOD_EXCLUDE_COUNT, article_id, photo_id)) res = api_method_exclude_count(article_id, photo_id) return Response(json.dumps({'exclude-count': res}), mimetype='application/json') else: return Response(json.dumps(['no-content']), mimetype='application/json')
def insert(self, obj_name, **kwargs): """ Method to insert rows in database :param name: object to persist :param **kwargs: field values :return: boolean indicating status of action """ if not self.session: log.error('No session') return False try: log.info('Attempting to insert row in schema "%s": "%s"' % ( obj_name, str([key + ':' + str(kwargs[key])[:100] for key in kwargs]))) self.session.add(getattr(schema, obj_name)(**kwargs)) self.session.commit() return True except Exception as e: log.error('Failed to insert row: "%s"' % e.message) return False
def process_photos(article_id, photos, user_obj): """ Handles linking photo results with the model and returns a list of Flickr photo ids to pass to templating :param article_id: int; article id :param photos: list of photos :param user_obj: User; user object for request :return: List of Flickr photo ids """ photo_ids = [] for photo in photos: # Ensure that each photo is modeled with PhotoModel() as pm: photo_obj = pm.get_photo(photo['photo_id'], article_id) if not photo_obj: log.info('Processing photo: "%s"' % str(photo)) if pm.insert_photo(photo['photo_id'], article_id): photo_obj = pm.get_photo( photo['photo_id'], article_id) if not photo_obj: log.error('DB Error: Could not retrieve or ' 'insert: "%s"' % str(photo)) continue else: log.error('Couldn\'t insert photo: "%s"' % ( photo['photo_id'])) photo['id'] = photo_obj._id photo['votes'] = photo_obj.votes # Retrieve like data with LikeModel() as lm: if lm.get_like(article_id, photo_obj._id, user_obj.get_id()): photo['like'] = True else: photo['like'] = False photo_ids.append(photo['photo_id']) return photo_ids
'token': edittoken, 'filename': filename, } # DISABLED # # if async: # data['asyncdownload'] = 1 # Send request response = requests.post(COMMONS_API_URL, data, auth=auth1, headers=header) if response.status_code != requests.codes.ok: log.error('Bad response status: "%s"' % response.status_code) else: # TODO - Update the upload model log.info('upload photo url: %s' % photo_url) log.info('upload edit token: %s' % str(edittoken)) pass return response def api_fetch_edit_token(token): """Get an edit token""" auth1 = OAuth1( settings.MW_CLIENT_KEY, client_secret=settings.MW_CLIENT_SECRET, resource_owner_key=token.key, resource_owner_secret=token.secret ) header = {'User-Agent': USER_AGENT} data = {
def mashup(): DataIORedis().connect() mysql_inst = DataIOMySQL() mysql_inst.connect() # Check for POST otherwise GET refresh = False if request.form: article = str(request.form['article']).strip() article = '_'.join(article.split()) log.debug('Processing POST - ' + article) else: article = str(request.args.get(settings.GET_VAR_ARTICLE)).strip() if 'refresh' in request.args: refresh = True article = '_'.join(article.split()) log.debug('Processing GET - ' + article) # Fetch article count and stored body (if exists) article_count = get_article_count() body = get_article_stored_body(article) if not body or refresh: # Calls to Wiki & Flickr APIs try: wiki = call_wiki(article) except WikiAPICallError as e: return render_template(e.template, error=e.message) try: res_json = call_flickr(article) except FlickrAPICallError as e: return render_template(e.template, error=e.message) # Extract photo data photos = get_flickr_photos(res_json) if not photos: render_template('index.html', error="Couldn't find any photos " "for '{0}'!".format(article)) # 1. Fetch the max article - Refresh periodically # 2. Remove a random article and replace, ensure that max has # been fetched # 3. Article insertion and ORM fetch # 4. rank photos according to UGC # 5. Photo & markup parsing # 6. Article content insertion and ORM fetch max_aid = get_max_article_id() manage_article_storage(max_aid, article_count) article_id, insert_ok = handle_article_insert(article, wiki.pageid) photos = order_photos_by_rank(article_id, photos) page_content = prep_page_content(article_id, article, wiki, photos, User(current_user.get_id())) if insert_ok: handle_article_content_insert(article_id, page_content, not body) else: page_content = json.loads(body, object_hook=_decode_dict) # refresh the user id page_content['user_id'] = User(current_user.get_id()).get_id() # Update last_access with ArticleModel() as am: am.update_last_access(page_content['article_id']) log.info('Rendering article "%s"' % article) return render_template('mashup.html', **page_content)
def upload_complete(): """POST, Renders the page for completing upload to mediawiki via api :return: template for view """ success = True msg = '' # Attempt api upload uid = hmac(User(current_user.get_id()).get_id()) log.info('Attempting upload to Commons for user: '******'article'] filename = request.form['filename'] photourl = request.form['photourl'] flickr_photo_id = request.form['flickr_photo_id'] articleurl = settings.SITE_URL + '/mashup?article=' + article # Obtain access token if it exists acc_token = None try: acc_token = mw.get_serialized(settings.MWOAUTH_ACCTOKEN_PKL_KEY, uid) except IOError: msg = 'No mediawiki token for your user. See <a href="%s">MWOauth</a>' % ( settings.SITE_URL + '/mwoauth') log.info('No mediawiki token for "%s"' % str(uid)) success = False # If access token was successfully fetched talk to commons api if success: response = mw.api_upload_url(request.form['photourl'], acc_token, filename) # Validate the response if response.status_code != requests.codes.ok: success = False msg = str(response.status_code) elif 'error' in response.json(): success = False msg = response.json()['error']['info'] else: success = True msg = 'OK' # Determine if the photo has already been uploaded to commons with UploadsModel() as um: if um.get_upload(flickr_photo_id): msg = 'This photo has already been uploaded.' success = False # Ensure that upload model is updated if success: with ArticleModel() as am: article_data = am.get_article_by_name(article) with PhotoModel() as pm: photo_data = pm.get_photo(flickr_photo_id, article_data.id) with UploadsModel() as um: um.insert_upload(photo_data.id, flickr_photo_id, article_data.id, uid) log.info('UPLOAD RESPONSE: ' + str(response.json())) return render_template('upload_complete.html', success=success, articleurl=articleurl, article=article, photourl=photourl, apierror=msg )