def manage_article_storage(max_article_id, article_count): """ Handle the storage of new articles :param max_article_id: int; article id :param article_count: int; total count of articles :return: bool; success """ if article_count >= settings.MYSQL_MAX_ROWS: if max_article_id: # TODO - CHANGE THIS be careful, could iterate many times article_removed = False attempts = 0 while not article_removed \ or attempts > settings.MAX_RETRIES_FOR_REMOVE: attempts += 1 article_id = random.randint(0, int(max_article_id)) with ArticleModel() as am: log.info('Removing article id: ' + str(article_id)) try: am.delete_article(article_id) article_removed = True except UnmappedInstanceError: continue else: log.error('Could not determine a max article id.') return True
def call_wiki(article): """Handles calling the wikipedia API via the local model :param article: :return: """ try: return wikipedia.page(article, preload=True) except DisambiguationError as e: raise WikiAPICallError( template='disambiguate.html', message='Received disambiguation list.', options=e.options ) except PageError as e: raise WikiAPICallError( template='index.html', message='Couldn\'t find the content for "%s".' % article, ) except (KeyError, TypeError) as e: log.error('Couldn\'t find %s: "%s"' % (article, str(e))) raise WikiAPICallError( template='index.html', message='Couldn\'t find the content for "%s".' % article, ) except Exception as e: log.error('Couldn\'t fetch "%s" from api: "%s"' % ( article, str(e))) raise WikiAPICallError( template='index.html', message='Underlying API could not be reached for "%s".' % article, )
def api_get_like(uid, pid, aid): """ Determines the like-glyph value for the given triplet :param uid: Flickipedia user id :param pid: Flickipedia photo id :param aid: Flickipedia article id :return: 'Like' row if exists, None otherwise """ # TODO - USE MODELS io = DataIOMySQL() io.connect() schema_obj = getattr(schema, 'Likes') # Query to extract res = io.session.query(schema_obj, schema_obj.is_set).filter( schema_obj.photo_id == pid, schema_obj.article_id == aid, schema_obj.user_id == uid ).limit(1).all() # Clean up connections io.sess.close() io.engine.dispose() if len(res) == 0: log.error('REST \'api_get_glyph\': Couldn\'t find (' 'user="******", photo_id=%s, article_id=%s)' % ( uid, pid, aid)) return None else: return res[0]
def delete(self, **kwargs): if self.conn: try: return self.conn.delete(kwargs["key"]) except KeyError as e: log.error("Missing param -> {0}".format(e.message)) return False else: log.error("No redis connection.") return False
def read(self, key): if self.conn: try: return self.conn.get(key) except KeyError as e: log.error("Missing param -> {0}".format(e.message)) return False else: log.error("No redis connection.") return False
def check_password(self, password): if self.pw_hash: try: password = escape(str(password)) return self.pw_hash == hmac(password) except (TypeError, NameError) as e: log.error(__name__ + ' :: Hash check error - ' + e.message) return False else: return False
def get_article_count(self): """ Fetches the number of articles indexed in the DB :return: Integer value of article count """ schema_obj = getattr(schema, 'Article') query_obj = self.io.session.query(func.count( schema_obj._id).label('cnt')) res = self.alchemy_fetch_validate(query_obj) if len(res) > 0: return res[0].cnt else: log.error('Couldn\'t get the count of articles.') return 0
def get_max_id(self): """ Fetch maximum article id :return: int id """ schema_obj = getattr(schema, 'Article') query_obj = self.io.session.query(func.max(schema_obj._id).label('id')) res = self.alchemy_fetch_validate(query_obj) if len(res) > 0: return res[0].id else: log.error('Couldn\'t get max article id.') return 0
def create_table(self, obj_name): """ Method for table creation :param name: schema object name :return: boolean indicating status """ if hasattr(schema, obj_name): getattr(schema, obj_name).__table__.create(bind=self.engine) return True else: log.error('Schema object not found for "%s"' % obj_name) return False
def mwoauth_complete(): """Complete the mw-auth for this user by storing their access token :return: template for view """ url = escape(str(request.form['callback_url'].strip())) id = User(current_user.get_id()).get_id() query_params = url.split('?')[-1] success = True try: get_MW_access_token(id, query_params) except Exception as e: log.error('Failed to generate access token: "%s"' % e.message) success = False return render_template('mwoauth_complete.html', success=success)
def fetch_row(self, tbl, col, value): """ Fetch a row by id :param tbl: str, table name :param col: str, column name :param value: *, value on whih to filter """ schema_obj = getattr(schema, tbl) try: return self.session.query(schema_obj).filter( getattr(schema_obj, col) == value) except Exception as e: log.error('Couldn\'t filter row: "%s"' % e.message) return []
def handle_article_insert(article, wiki_page_id): """ Handle insertion of article meta data :param article_id: int; article id :return: int, bool; success """ with ArticleModel() as am: if am.insert_article(article, wiki_page_id): article_obj = am.get_article_by_name(article) article_id = article_obj._id success = True else: log.error('Couldn\'t insert article: "%s"' % article) article_id = -1 success = False return article_id, success
def delete(self, qry_obj): """ Method to delete rows from database :param qry_obj: object to delete :return: boolean indicating status of action """ if not self.session: log.error('No session') return False try: self.session.delete(qry_obj) self.session.commit() return True except Exception as e: log.error('Failed to delete row "%s": "%s"' % (str(qry_obj), e.message())) return False
def call_flickr(search_str): """Handles calling the flickr API via the local model :param search_str: :return: """ try: res = flickr.call('photos_search', {'text': ' '.join(search_str.split('_')), 'format': 'json', 'sort': 'relevance', 'license': "4,5,7,8" }) except Exception as e: log.error('Flickr api.photos.search failed with: "%s"' % e.message) raise FlickrAPICallError( template='index.html', message='Flickr search request failed "%s"' % search_str, ) return json.loads(res[14:-1])
def alchemy_fetch_validate(self, sqlAlchemyQryObj, retType = RET_TYPE_ALLROWS): """ Fault tolerance around query execution in sql alachemy :param schema_obj: :return: """ retries = 0 while retries < NUM_SQL_RETRIES: try: if retType == RET_TYPE_ALLROWS: return sqlAlchemyQryObj.all() elif retType == RET_TYPE_COUNT: return sqlAlchemyQryObj.count() elif retType == RET_TYPE_FIRSTROW: return sqlAlchemyQryObj[0] except OperationalError: log.error('Failed to fetch article, trying again.') retries += 1 time.sleep(0.5) return []
def insert(self, obj_name, **kwargs): """ Method to insert rows in database :param name: object to persist :param **kwargs: field values :return: boolean indicating status of action """ if not self.session: log.error('No session') return False try: log.info('Attempting to insert row in schema "%s": "%s"' % ( obj_name, str([key + ':' + str(kwargs[key])[:100] for key in kwargs]))) self.session.add(getattr(schema, obj_name)(**kwargs)) self.session.commit() return True except Exception as e: log.error('Failed to insert row: "%s"' % e.message) return False
def call(methodname, params=None): """ Invokes API method """ log.debug('API KEY = {0}, SECRET = {1}'.format( settings.FLICKR_API_KEY, settings.FLICKR_API_SECRET )) flickr = flickrapi.FlickrAPI(settings.FLICKR_API_KEY, secret=settings.FLICKR_API_SECRET) try: # Extract the api method log.debug('Calling method - "%s"' % methodname) method = getattr(flickr, methodname) except Exception: log.error('No such API method.') return try: if not params: return method(format='json') else: return method(**params) except urllib2.HTTPError as e: log.error('Could not reach service.') except Exception as e: log.error(e.message) return None
def process_photos(article_id, photos, user_obj): """ Handles linking photo results with the model and returns a list of Flickr photo ids to pass to templating :param article_id: int; article id :param photos: list of photos :param user_obj: User; user object for request :return: List of Flickr photo ids """ photo_ids = [] for photo in photos: # Ensure that each photo is modeled with PhotoModel() as pm: photo_obj = pm.get_photo(photo['photo_id'], article_id) if not photo_obj: log.info('Processing photo: "%s"' % str(photo)) if pm.insert_photo(photo['photo_id'], article_id): photo_obj = pm.get_photo( photo['photo_id'], article_id) if not photo_obj: log.error('DB Error: Could not retrieve or ' 'insert: "%s"' % str(photo)) continue else: log.error('Couldn\'t insert photo: "%s"' % ( photo['photo_id'])) photo['id'] = photo_obj._id photo['votes'] = photo_obj.votes # Retrieve like data with LikeModel() as lm: if lm.get_like(article_id, photo_obj._id, user_obj.get_id()): photo['like'] = True else: photo['like'] = False photo_ids.append(photo['photo_id']) return photo_ids
def api_set_like(uid, pid, aid): """ Toggles the like-glyph value for the given triplet :param uid: Flickipedia user id :param pid: Flickipedia photo id :param aid: Flickipedia article id :return: True on success, False otherwise """ # TODO - USE MODELS io = DataIOMySQL() io.connect() result = api_get_like(uid, pid, aid) # toggle and set new value (delete row if it doesn't exist) if result: # io.update false try: io.delete(result) except Exception as e: log.error(' "%s"' % e.message) return False else: # io.update true try: io.insert('Like', user_id=uid, photo_id=pid, article_id=aid) except Exception as e: log.error(' "%s"' % e.message) return False # Clean up connections io.sess.close() io.engine.dispose() return True
def get_flickr_photos(flickr_json): """ Retrience Flickr photo content from Flickr API :param article: str; article name :return: list; list of Flickr photo json """ photos = [] for i in xrange(settings.NUM_PHOTOS_TO_FETCH): try: photos.append( { 'owner': flickr_json['photos']['photo'][i]['owner'], 'photo_id': flickr_json['photos']['photo'][i]['id'], 'farm': flickr_json['photos']['photo'][i]['farm'], 'server': flickr_json['photos']['photo'][i]['server'], 'title': flickr_json['photos']['photo'][i]['title'], 'secret': flickr_json['photos']['photo'][i]['secret'], }, ) except (IndexError, KeyError) as e: log.error('No more photos to process for: - "%s"' % (e.message)) log.debug('Photo info: %s' % (str(photos))) return photos
def api_fetch_edit_token(token): """Get an edit token""" auth1 = OAuth1( settings.MW_CLIENT_KEY, client_secret=settings.MW_CLIENT_SECRET, resource_owner_key=token.key, resource_owner_secret=token.secret ) header = {'User-Agent': USER_AGENT} data = { 'format': 'json', 'action': 'tokens', 'type': 'edit', } # Fetch token, send request response = requests.get(COMMONS_API_URL, params=data, auth=auth1, headers=header) if response.status_code != requests.codes.ok: log.error('Bad response status: "%s"' % response.status_code) try: return response.json()['tokens']['edittoken'] except (ValueError, KeyError) as e: log.error("Missing Edit token: %s" % e.message) return None
'format': 'json', 'action': 'upload', 'url': photo_url, 'token': edittoken, 'filename': filename, } # DISABLED # # if async: # data['asyncdownload'] = 1 # Send request response = requests.post(COMMONS_API_URL, data, auth=auth1, headers=header) if response.status_code != requests.codes.ok: log.error('Bad response status: "%s"' % response.status_code) else: # TODO - Update the upload model log.info('upload photo url: %s' % photo_url) log.info('upload edit token: %s' % str(edittoken)) pass return response def api_fetch_edit_token(token): """Get an edit token""" auth1 = OAuth1( settings.MW_CLIENT_KEY, client_secret=settings.MW_CLIENT_SECRET, resource_owner_key=token.key, resource_owner_secret=token.secret