def connect(self, log=False):
""" dialect+driver://username:password@host:port/database """
if self.driver:
connect_str = '{0}+{1}://{2}:{3}@{4}/{5}'.format(
self.dialect,
self.driver,
self.user,
self.pwrd,
self.host,
self.db,
)
else:
connect_str = '{0}://{1}:{2}@{3}/{4}'.format(
self.dialect,
self.user,
self.pwrd,
self.host,
self.db,
)
if log:
log.info('Establishing connection to "%s://%s@%s/%s"' % (
self.dialect,
self.user,
self.host,
self.db
))
self.engine = create_engine(connect_str)
self.make_session()
def manage_article_storage(max_article_id, article_count):
"""
Handle the storage of new articles
:param max_article_id: int; article id
:param article_count: int; total count of articles
:return: bool; success
"""
if article_count >= settings.MYSQL_MAX_ROWS:
if max_article_id:
# TODO - CHANGE THIS be careful, could iterate many times
article_removed = False
attempts = 0
while not article_removed \
or attempts > settings.MAX_RETRIES_FOR_REMOVE:
attempts += 1
article_id = random.randint(0, int(max_article_id))
with ArticleModel() as am:
log.info('Removing article id: ' + str(article_id))
try:
am.delete_article(article_id)
article_removed = True
except UnmappedInstanceError:
continue
else:
log.error('Could not determine a max article id.')
return True
def login():
if request.method == 'POST' and 'username' in request.form:
username = escape(str(request.form['username'].strip()))
passwd = escape(str(request.form['password'].strip()))
remember = request.form.get('remember', 'no') == 'yes'
log.info('Attempting login for "%s"' % username)
with UserModel() as um:
user = um.fetch_user_by_name(username)
if not user:
log.info('On login - User not found "%s"' % username)
flash('Login failed.')
return render_template('login.html')
user_ref = User(user._id)
user_ref.authenticate(passwd)
if user_ref.is_authenticated():
login_user(user_ref, remember=remember)
flash('Logged in.')
return redirect(request.args.get('next')
or url_for('home'))
else:
flash('Login failed.')
return render_template('login.html')
def fetch_user_by_name(self, handle):
query_obj = self.io.session.query(schema.User).filter(
schema.User.handle == handle)
try:
return self.alchemy_fetch_validate(query_obj, RET_TYPE_FIRSTROW)
except (KeyError, IndexError) as e:
log.info('User not found "%s": %s' % (handle, e.message))
return None
def get_serialized(type, key):
"""Retrieves the obj from pickle file for this type/key
:param type: object type to pickle
:param key: unique id
:return: serialized object
"""
fname = getPicklerFilename(type, key)
with open(fname, 'rb') as f:
log.info('Fetching pickle: ' + fname)
return cPickle.load(f)
def set_serialized(obj, type, key):
"""Sets the pickle file for this type/key
:param obj: object to serialize
:param type: object type to pickle
:param key: unique id
"""
fname = getPicklerFilename(type, key)
with open(fname, 'wb') as f:
log.info('Setting pickle: ' + fname)
cPickle.dump(obj, f)
def get_upload(self, flickr_photo_id):
"""
Retrieve a photo by its flickr id
"""
log.info('Fetching photo by flickr_photo_id: %s' % flickr_photo_id)
schema_obj = getattr(schema, 'Upload')
query_obj = self.io.session.query(schema_obj).filter(
schema_obj.flickr_photo_id == flickr_photo_id)
res = self.alchemy_fetch_validate(query_obj)
if len(res) > 0:
return res[0]
else:
return None
def init_views():
for key in view_list:
# Add maintenance wrapper
# view_list[key] = maintenance(view_list[key])
# wrap methods for login requirement
if key not in views_with_anonymous_access:
view_list[key] = login_required(view_list[key])
for key in route_deco:
route = route_deco[key]
view_method = view_list[key]
view_list[key] = route(view_method)
log.info(__name__ + ' :: Registered views - {0}'.format(str(view_list)))
def get_article_stored_body(article):
"""
Fetch corresponding article object
:param article: str; article name
:return: json, Article; stored page content, corresponding
article model object
"""
with ArticleModel() as am:
article_obj = am.get_article_by_name(article)
try:
with ArticleContentModel() as acm:
body = acm.get_article_content(article_obj._id).markup
except Exception as e:
log.info('Article markup not found: "%s"' % e.message)
body = ''
return body
def handle_photo_integrate(photos, html, article):
"""
Integrate photo link tags into content. This walks through each section
header and inserts an image below the header.
:param photos: List of photo meta-info
:param html: Wiki-html
:return: modified content
"""
soup = BeautifulSoup(html)
photo_index = 0
# Embed Title photo
lf = '<div style="clear:both;"> </div>'
try:
tag = embed_photo_content(
article,
photo_index,
photos[photo_index],
soup,
TITLE_PHOTO_SIZE_X,
TITLE_PHOTO_SIZE_Y
)
except (ValueError, KeyError, IndexError):
log.info('In parse no photos found')
return html
html = str(tag) + lf + lf + html
photo_index += 1
# Embed section photos
headers = soup.findAll('h2')
headers.extend(soup.findAll('h3'))
for node in headers:
if len(photos) > photo_index:
tag = embed_photo_content(
article,
photo_index,
photos[photo_index],
soup
)
html = html.replace(str(node), str(node) + str(tag))
photo_index += 1
else:
break
return html
def api(method):
"""REST interface for flickipedia - swtches on method calls"""
# Extract photo-id, article-id, user-id
article_id = request.args.get('article-id')
user_id = request.args.get('user-id')
photo_id = request.args.get('photo-id')
if method == API_METHOD_ENDORSE_EVENT:
log.info('On %s getting (article, user, photo) = (%s, %s, %s)' % (
API_METHOD_ENDORSE_EVENT, article_id, user_id, photo_id))
api_method_endorse_event(article_id, user_id, photo_id)
return Response(json.dumps(['endorse-event']), mimetype='application/json')
elif method == API_METHOD_ENDORSE_FETCH:
log.info('On %s getting (article, user, photo) = (%s, %s, %s)' % (
API_METHOD_ENDORSE_FETCH, article_id, user_id, photo_id))
res = api_method_endorse_fetch(article_id, user_id, photo_id)
return Response(json.dumps({'endorse-fetch': res}), mimetype='application/json')
elif method == API_METHOD_EXCLUDE_EVENT:
api_method_exclude_event(article_id, user_id, photo_id)
return Response(json.dumps(['exclude-event']), mimetype='application/json')
elif method == API_METHOD_EXCLUDE_FETCH:
log.info('On %s getting (article, user, photo) = (%s, %s, %s)' % (
API_METHOD_EXCLUDE_FETCH, article_id, user_id, photo_id))
res = api_method_exclude_fetch(article_id, user_id, photo_id)
return Response(json.dumps({'exclude-fetch': res}), mimetype='application/json')
elif method == API_METHOD_ENDORSE_COUNT:
log.info('On %s getting (article, photo) = (%s, %s)' % (
API_METHOD_EXCLUDE_COUNT, article_id, photo_id))
res = api_method_endorse_count(article_id, photo_id)
return Response(json.dumps({'endorse-count': res}), mimetype='application/json')
elif method == API_METHOD_EXCLUDE_COUNT:
log.info('On %s getting (article, photo) = (%s, %s)' % (
API_METHOD_EXCLUDE_COUNT, article_id, photo_id))
res = api_method_exclude_count(article_id, photo_id)
return Response(json.dumps({'exclude-count': res}), mimetype='application/json')
else:
return Response(json.dumps(['no-content']), mimetype='application/json')
def insert(self, obj_name, **kwargs):
"""
Method to insert rows in database
:param name: object to persist
:param **kwargs: field values
:return: boolean indicating status of action
"""
if not self.session:
log.error('No session')
return False
try:
log.info('Attempting to insert row in schema "%s": "%s"' % (
obj_name, str([key + ':' + str(kwargs[key])[:100] for key in kwargs])))
self.session.add(getattr(schema, obj_name)(**kwargs))
self.session.commit()
return True
except Exception as e:
log.error('Failed to insert row: "%s"' % e.message)
return False
def process_photos(article_id, photos, user_obj):
"""
Handles linking photo results with the model and returns a list of
Flickr photo ids to pass to templating
:param article_id: int; article id
:param photos: list of photos
:param user_obj: User; user object for request
:return: List of Flickr photo ids
"""
photo_ids = []
for photo in photos:
# Ensure that each photo is modeled
with PhotoModel() as pm:
photo_obj = pm.get_photo(photo['photo_id'], article_id)
if not photo_obj:
log.info('Processing photo: "%s"' % str(photo))
if pm.insert_photo(photo['photo_id'], article_id):
photo_obj = pm.get_photo(
photo['photo_id'], article_id)
if not photo_obj:
log.error('DB Error: Could not retrieve or '
'insert: "%s"' % str(photo))
continue
else:
log.error('Couldn\'t insert photo: "%s"' % (
photo['photo_id']))
photo['id'] = photo_obj._id
photo['votes'] = photo_obj.votes
# Retrieve like data
with LikeModel() as lm:
if lm.get_like(article_id, photo_obj._id,
user_obj.get_id()):
photo['like'] = True
else:
photo['like'] = False
photo_ids.append(photo['photo_id'])
return photo_ids
'token': edittoken,
'filename': filename,
}
# DISABLED
#
# if async:
# data['asyncdownload'] = 1
# Send request
response = requests.post(COMMONS_API_URL, data, auth=auth1, headers=header)
if response.status_code != requests.codes.ok:
log.error('Bad response status: "%s"' % response.status_code)
else:
# TODO - Update the upload model
log.info('upload photo url: %s' % photo_url)
log.info('upload edit token: %s' % str(edittoken))
pass
return response
def api_fetch_edit_token(token):
"""Get an edit token"""
auth1 = OAuth1(
settings.MW_CLIENT_KEY,
client_secret=settings.MW_CLIENT_SECRET,
resource_owner_key=token.key,
resource_owner_secret=token.secret
)
header = {'User-Agent': USER_AGENT}
data = {
def mashup():
DataIORedis().connect()
mysql_inst = DataIOMySQL()
mysql_inst.connect()
# Check for POST otherwise GET
refresh = False
if request.form:
article = str(request.form['article']).strip()
article = '_'.join(article.split())
log.debug('Processing POST - ' + article)
else:
article = str(request.args.get(settings.GET_VAR_ARTICLE)).strip()
if 'refresh' in request.args:
refresh = True
article = '_'.join(article.split())
log.debug('Processing GET - ' + article)
# Fetch article count and stored body (if exists)
article_count = get_article_count()
body = get_article_stored_body(article)
if not body or refresh:
# Calls to Wiki & Flickr APIs
try:
wiki = call_wiki(article)
except WikiAPICallError as e:
return render_template(e.template, error=e.message)
try:
res_json = call_flickr(article)
except FlickrAPICallError as e:
return render_template(e.template, error=e.message)
# Extract photo data
photos = get_flickr_photos(res_json)
if not photos:
render_template('index.html', error="Couldn't find any photos "
"for '{0}'!".format(article))
# 1. Fetch the max article - Refresh periodically
# 2. Remove a random article and replace, ensure that max has
# been fetched
# 3. Article insertion and ORM fetch
# 4. rank photos according to UGC
# 5. Photo & markup parsing
# 6. Article content insertion and ORM fetch
max_aid = get_max_article_id()
manage_article_storage(max_aid, article_count)
article_id, insert_ok = handle_article_insert(article, wiki.pageid)
photos = order_photos_by_rank(article_id, photos)
page_content = prep_page_content(article_id, article, wiki, photos,
User(current_user.get_id()))
if insert_ok:
handle_article_content_insert(article_id, page_content, not body)
else:
page_content = json.loads(body, object_hook=_decode_dict)
# refresh the user id
page_content['user_id'] = User(current_user.get_id()).get_id()
# Update last_access
with ArticleModel() as am:
am.update_last_access(page_content['article_id'])
log.info('Rendering article "%s"' % article)
return render_template('mashup.html', **page_content)
def upload_complete():
"""POST, Renders the page for completing upload to mediawiki via api
:return: template for view
"""
success = True
msg = ''
# Attempt api upload
uid = hmac(User(current_user.get_id()).get_id())
log.info('Attempting upload to Commons for user: '******'article']
filename = request.form['filename']
photourl = request.form['photourl']
flickr_photo_id = request.form['flickr_photo_id']
articleurl = settings.SITE_URL + '/mashup?article=' + article
# Obtain access token if it exists
acc_token = None
try:
acc_token = mw.get_serialized(settings.MWOAUTH_ACCTOKEN_PKL_KEY, uid)
except IOError:
msg = 'No mediawiki token for your user. See <a href="%s">MWOauth</a>' % (
settings.SITE_URL + '/mwoauth')
log.info('No mediawiki token for "%s"' % str(uid))
success = False
# If access token was successfully fetched talk to commons api
if success:
response = mw.api_upload_url(request.form['photourl'], acc_token, filename)
# Validate the response
if response.status_code != requests.codes.ok:
success = False
msg = str(response.status_code)
elif 'error' in response.json():
success = False
msg = response.json()['error']['info']
else:
success = True
msg = 'OK'
# Determine if the photo has already been uploaded to commons
with UploadsModel() as um:
if um.get_upload(flickr_photo_id):
msg = 'This photo has already been uploaded.'
success = False
# Ensure that upload model is updated
if success:
with ArticleModel() as am:
article_data = am.get_article_by_name(article)
with PhotoModel() as pm:
photo_data = pm.get_photo(flickr_photo_id, article_data.id)
with UploadsModel() as um:
um.insert_upload(photo_data.id, flickr_photo_id, article_data.id, uid)
log.info('UPLOAD RESPONSE: ' + str(response.json()))
return render_template('upload_complete.html',
success=success,
articleurl=articleurl,
article=article,
photourl=photourl,
apierror=msg
)
