def main():
upload_only = False
if len(sys.argv) > 1 and sys.argv[1] == '--upload-only':
upload_only = True
update_q = Queue('update', connection=Redis(host='redis'))
upload_q = Queue('upload', connection=Redis(host='redis'))
if (update_q.count > 0 or upload_q.count > 0):
print('Queues are not empty. Refusing to add more work.')
return
wikidb = wiki_connect()
for project_name in logic_project.project_names_to_update(wikidb):
if upload_only:
print('Enqueuing upload %s' % project_name)
upload_q.enqueue(tables.upload_project_table,
project_name,
job_timeout=constants.JOB_TIMEOUT)
else:
print('Enqueuing update %s' % project_name)
update_job = update_q.enqueue(logic_project.update_project_by_name,
project_name,
job_timeout=constants.JOB_TIMEOUT)
print('Enqueuing upload (dependent) %s' % project_name)
upload_q.enqueue(tables.upload_project_table,
project_name,
depends_on=update_job,
job_timeout=constants.JOB_TIMEOUT)
print('Enqueuing log upload (dependent) %s' % project_name)
upload_q.enqueue(logs.update_log_page_for_project,
project_name,
depends_on=update_job,
job_timeout=constants.JOB_TIMEOUT)
def update_log_page_for_project(project_name):
wikidb = wiki_connect()
wp10db = wp10_connect()
try:
log_map = calculate_logs_to_update(wikidb, wp10db, project_name)
edits = generate_log_edits(wikidb, wp10db, project_name, log_map)
p = api.get_page(log_page_name(project_name))
header = ('{{Log}}\n'
'<noinclude>[[Category:%s articles by quality]]</noinclude>\n' %
project_name.decode('utf-8').replace('_', ' '))
if len(edits) == 0:
today = get_current_datetime()
from_dt = get_current_datetime() - timedelta(days=7)
update = ("%s'''There were no logs for this project from %s - %s.'''" %
(header, from_dt.strftime(LOG_DATE_FORMAT),
today.strftime(LOG_DATE_FORMAT)))
else:
update = header + '\n'.join(edits)
i = 1
while len(update) > 2048 * 1024:
update = header + '\n'.join(edits[:-1 * i])
i += 1
if i == len(edits):
update = (header + 'Sorry, all of the logs for this date were too '
'large to upload.')
api.save_page(p, update, 'Update logs for past 7 days')
finally:
if wikidb:
wikidb.close()
if wp10db:
wp10db.close()
def update_project_by_name(project_name, track_progress=False):
wp10db = wp10_connect()
wikidb = wiki_connect()
redis = redis_connect()
logging.basicConfig(level=logging.INFO)
logging.getLogger('mwclient').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('requests_oauthlib').setLevel(logging.CRITICAL)
logging.getLogger('oauthlib').setLevel(logging.CRITICAL)
try:
project = get_project_by_name(wp10db, project_name)
if not project:
project = Project(p_project=project_name,
p_timestamp=GLOBAL_TIMESTAMP_WIKI)
update_project(wikidb,
wp10db,
project,
redis=redis,
track_progress=track_progress)
if track_progress:
redis.expire(_project_progress_key(project_name), 600)
finally:
wp10db.close()
wikidb.close()
def main():
wikidb = wiki_connect()
wp10db = wp10_connect()
logging.basicConfig(level=logging.INFO)
for project_name in logic_project.project_names_to_update(wikidb):
logic_project.update_global_articles_for_project_name(wp10db, project_name)
def enqueue_all_projects(redis):
update_q, upload_q = _get_queues(redis)
if (update_q.count > 0 or upload_q.count > 0):
logger.error('Queues are not empty. Refusing to add more work.')
return
wikidb = wiki_connect()
for project_name in logic_project.project_names_to_update(wikidb):
enqueue_project(project_name, update_q, upload_q)
def update_project_by_name(project_name):
wp10db = wp10_connect()
wikidb = wiki_connect()
logging.basicConfig(level=logging.INFO)
logging.getLogger('mwclient').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('requests_oauthlib').setLevel(logging.CRITICAL)
logging.getLogger('oauthlib').setLevel(logging.CRITICAL)
try:
project = get_project_by_name(wp10db, project_name)
if not project:
logger.error('No project with name: %s', project_name)
return
update_project(wikidb, wp10db, project)
finally:
wp10db.close()
wikidb.close()