def list_editor_submit(listname, request):
meta = request.values['meta'].lstrip('##')
articles = [a.strip() for a in request.values['articles'].split('\n')]
resolve = request.values.get('resolve')
alm = ArticleListManager()
alm.append_action(listname, meta, articles)
if resolve:
alm.resolve_the_unresolved(listname)
return {
'success': True
}
def list_editor(listname):
alm = ArticleListManager()
article_list = alm.load_list(listname)
ret = {'name': listname,
'meta': article_list.file_metadata_string,
'actions': [],
'article_set': article_list.get_articles()}
for action in article_list.actions:
ret['actions'].append({'meta': {'action': action.action},
'articles': action.articles})
return ret
def list_editor_remove(listname, request):
article_list = []
for article_name in request.values.keys():
if request.values[article_name] == 'remove' \
and article_name != '_list_name':
article_list.append(article_name)
meta = dumps({'action': 'exclude',
'date': time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()),
'term': ['manual action'],
'source': 'http://en.wikipedia.org/w/api.php'})
alm = ArticleListManager()
alm.append_action(listname, meta, article_list)
return {
'success': True
}
def __init__(self, env_or_path=None, debug=True):
self.alm = ArticleListManager(env_or_path)
if not env_or_path or isinstance(env_or_path, basestring):
self.env = None
defpath = env_or_path or os.getenv('WOMP_FETCH_HOME') \
or os.getcwd()
self._home = defpath
else:
self.env = env_or_path
self._home = self.env.fetch_home
self.debug = debug
self.inputs = DEFAULT_INPUTS
def list_create(listname, request):
alm = ArticleListManager()
try:
alm.create(listname)
thelist = alm.load_list(listname)
except IOError as e:
return {
'error': str(e),
'code': 409
}
except ValueError as e:
return {
'error': str(e),
'code': 400
}
return {
'name': listname,
'articles': len(thelist._get_unresolved_articles()),
'actions': len(thelist.actions),
'date': thelist.file_metadata.get('date', 'new')
}
class FetchManager(object):
def __init__(self, env_or_path=None, debug=True):
self.alm = ArticleListManager(env_or_path)
if not env_or_path or isinstance(env_or_path, basestring):
self.env = None
defpath = env_or_path or os.getenv('WOMP_FETCH_HOME') \
or os.getcwd()
self._home = defpath
else:
self.env = env_or_path
self._home = self.env.fetch_home
self.debug = debug
self.inputs = DEFAULT_INPUTS
def load_list(self, name):
article_list = self.alm.load_list(name)
if article_list is None:
raise ValueError('no article list named "%s"' % (name,))
return article_list.get_articles()
def create_job(self, article_list):
return FetchJob(article_list, self.inputs)
def fetch_list(self,
target_list_name,
no_dashboard=False,
no_pdb=False,
port=DEFAULT_PORT,
**kw):
if isinstance(target_list_name, list):
target_list_name = target_list_name[0] # dammit argparse
article_list = self.load_list(target_list_name)
fetch_job = self.create_job(article_list)
fetch_job.name = target_list_name # TODO: tmp
if not no_dashboard:
self.spawn_dashboard(fetch_job, port=port)
fetch_job.run()
if not no_pdb: # double negative for easier cli
import pdb
pdb.set_trace()
return
def spawn_dashboard(self, job, port):
print 'Spawning dashboard...'
sp_dashboard = create_fetch_dashboard(job)
tpool = ThreadPool(2)
tpool.spawn(sp_dashboard.serve,
use_reloader=False,
static_prefix='static',
port=port,
static_path=dashboard._STATIC_PATH)
def write(self):
if not self.results:
print 'no results, nothing to save'
return
print 'Writing...'
print 'Total results:', len(self.results)
print 'Incomplete results:', len([f for f in self.result_stats
if not f['is_successful']])
output_name = os.path.join(self._home, self.name + DEFAULT_EXT)
output_file = codecs.open(output_name, 'w', 'utf-8')
for result in self.results:
output_file.write(json.dumps(result, default=str))
output_file.write('\n')
output_file.close()
def article_list():
alm = ArticleListManager()
return {'article_lists': alm.get_all_list_dicts()}
def list_delete(listname, request):
alm = ArticleListManager()
alm.delete(listname)
return {
'success': True
}
