def parseWiki(con):
useragent = build_user_agent('l2wiki', 0.1,
'https://github.com/tm-calculate/l2wiki')
wiki = MediaWiki('http://l2central.info/c/api.php', user_agent=useragent)
cmcontinue = parseWikiPart(con, wiki)
while cmcontinue:
cmcontinue = parseWikiPart(con, wiki, cmcontinue)
def setUp(self):
self.user_agent = simplemediawiki.build_user_agent(
'python-simplemediawiki test suite', simplemediawiki.__version__,
'https://github.com/ianweller/python-simplemediawiki')
self.mw = simplemediawiki.MediaWiki('https://httpbin.org/',
user_agent=self.user_agent)
self.data = None
def test_build_user_agent(self):
user_agent = simplemediawiki.build_user_agent(
'python-simplemediawiki test suite', simplemediawiki.__version__,
'https://github.com/ilianaw/python-simplemediawiki')
self.assertTrue('python-simplemediawiki' in user_agent)
self.assertTrue(simplemediawiki.__version__ in user_agent)
self.assertTrue(
'github.com/ilianaw/python-simplemediawiki' in user_agent)
def create_UA(task=None, version=None, contact=None):
if task is None:
task = 'JonathanGoble'
if version is None:
version = '0.1.0a0'
if contact is None:
contact = contact_email
return simplemediawiki.build_user_agent(task, version, contact)
help=
"Clean the output directory after downloading, useful for removing pages deleted/moved on the wiki. Warning: any unknown files found in the output directory will be deleted!"
)
aparser.add_argument(
"--safe-filenames",
action="store_true",
help="Force using ASCII file names instead of the default Unicode.")
args = aparser.parse_args()
if args.force:
epoch = datetime.datetime.utcnow()
else:
# this should be the date of the latest incompatible change
epoch = datetime.datetime(2016, 3, 3, 18, 0, 0)
user_agent = build_user_agent(__file__, ArchWiki.__version__,
ArchWiki.__url__)
aw = ArchWiki.ArchWiki(user_agent=user_agent,
safe_filenames=args.safe_filenames)
optimizer = ArchWiki.Optimizer(aw, args.output_directory)
downloader = ArchWiki.Downloader(aw,
args.output_directory,
epoch,
cb_download=optimizer.optimize_url)
downloader.download_css()
aw.print_namespaces()
for ns in ["0", "4", "12", "14"]:
downloader.process_namespace(ns)
downloader.download_images()
user_agent = {
'application_name': 'ThanksLog',
'version': '0.0.2alpha',
'url': 'https://github.com/valepert/thankslog'
}
project = 'it'
api = 'w/api.php'
url = 'http://%s.wikipedia.org/%s' % (project, api)
thanksdir = 'thanks_%s' % project
lelimit = 500
n_times = 3
sleep_time = 60
wiki = simplemediawiki.MediaWiki(url)
simplemediawiki.build_user_agent(**user_agent)
#lastcontinue = u''
query = {
'action': 'query',
'list': 'logevents',
'letype': 'thanks',
'lelimit': lelimit,
'continue': u''
} # u'-||' } #, 'lecontinue': lastcontinue}
out = wiki.call(query)
results = out['query']['logevents']
dump_results(results, thanksdir)
cont = out['continue']
import ArchWiki
if __name__ == "__main__":
aparser = argparse.ArgumentParser(description="Download pages from Arch Wiki and optimize them for offline browsing")
aparser.add_argument("--output-directory", type=str, required=True, help="Path where the downloaded pages should be stored.")
aparser.add_argument("--force", action="store_true", help="Ignore timestamp, always download the page from the wiki.")
aparser.add_argument("--clean", action="store_true", help="Clean the output directory after downloading, useful for removing pages deleted/moved on the wiki. Warning: any unknown files found in the output directory will be deleted!")
aparser.add_argument("--safe-filenames", action="store_true", help="Force using ASCII file names instead of the default Unicode.")
args = aparser.parse_args()
if args.force:
epoch = datetime.datetime.utcnow()
else:
# this should be the date of the latest incompatible change
epoch = datetime.datetime(2014, 4, 12)
user_agent = build_user_agent(__file__, ArchWiki.__version__, ArchWiki.__url__)
aw = ArchWiki.ArchWiki(user_agent=user_agent, safe_filenames=args.safe_filenames)
optimizer = ArchWiki.Optimizer(aw, args.output_directory)
downloader = ArchWiki.Downloader(aw, args.output_directory, epoch, cb_download=optimizer.optimize_url)
aw.print_namespaces()
for ns in ["0", "4", "12", "14"]:
downloader.process_namespace(ns)
downloader.download_images()
downloader.download_css()
if args.clean:
downloader.clean_output_directory()
json.dump(result, entry)
user_agent = {'application_name': 'ThanksLog',
'version': '0.0.2alpha',
'url': 'https://github.com/valepert/thankslog'}
project = 'it'
api = 'w/api.php'
url = 'http://%s.wikipedia.org/%s' % (project, api)
thanksdir = 'thanks_%s' % project
lelimit = 500
n_times = 3
sleep_time = 60
wiki = simplemediawiki.MediaWiki(url)
simplemediawiki.build_user_agent(**user_agent)
#lastcontinue = u''
query = {'action': 'query', 'list': 'logevents', 'letype': 'thanks',
'lelimit': lelimit,
'continue': u''} # u'-||' } #, 'lecontinue': lastcontinue}
out = wiki.call(query)
results = out['query']['logevents']
dump_results(results, thanksdir)
cont = out['continue']
while results:
for _ in range(n_times):
try:
query = dict(query, **cont)
def setUp(self):
self.user_agent = simplemediawiki.build_user_agent(
'python-simplemediawiki test suite', simplemediawiki.__version__,
'https://github.com/ilianaw/python-simplemediawiki')
self.mw = simplemediawiki.MediaWiki('FIXME/api.php',
user_agent=self.user_agent)
import sys
from timestamper import *
import datetime
import time
import pprint
# Get wiki location
location = raw_input("Base URL to the wiki API (YOUR_WIKI_ROOT/api.php): ")
if (location[0:7].lower() != "http://"):
location = "http://" + location
wiki = MediaWiki(location)
if wiki.normalize_api_url() is None:
sys.exit("Invalid Wiki URL")
# Get login credetials
ua = build_user_agent("uturn", "0.1", "https://github.com/tomasreimers/wiki-uturn");
while True:
username = raw_input("Username: "******"Password: "******"Invalid login"
# Get date to revert to
print "When would you like to revert to (IN UTC)?"
year = int(raw_input("Year: "))
month = int(raw_input("Month: "))
day = int(raw_input("Day: "))
hour = int(raw_input("Hour: "))
minute = int(raw_input("Minute: "))
