def test_getLatestUploader(self):
"""Test getLatestUploader."""
page = pywikibot.FilePage(self.site, 'File:Albert Einstein.jpg')
latest = page.getLatestUploader()
self.assertOneDeprecation()
self.assertEqual(len(latest), 2)
self.assertIsInstance(latest[0], unicode)
self.assertIsInstance(latest[1], unicode)
def _finish_upload(self, chunk_size, file_name):
"""Finish the upload."""
# Finish/continue upload with the given file key
page = pywikibot.FilePage(self.site, 'MP_sounds-pwb.png')
self.site.upload(page, source_filename=file_name,
comment='pywikibot test', chunk_size=chunk_size,
ignore_warnings=True, report_success=False,
_file_key=self._file_key, _offset=self._offset)
def test_globalusage(self, key):
"""Test globalusage generator."""
page = pywikibot.FilePage(self.site, 'File:Example.jpg')
gen = page.globalusage(total=3)
pages = list(gen)
self.assertEqual(len(pages), 3)
self.assertTrue(any(isinstance(p), pywikibot.Page) for p in pages)
self.assertTrue(any(p.site != self.site for p in pages))
def test_png_chunked(self):
"""Test uploading a png in two chunks using Site.upload."""
page = pywikibot.FilePage(self.site, 'MP_sounds-pwb-chunked.png')
self.site.upload(page,
source_filename=self.sounds_png,
comment='pywikibot test',
ignore_warnings=True,
chunk_size=1024)
def test_png(self):
"""Test uploading a png using Site.upload."""
page = pywikibot.FilePage(self.site, 'MP_sounds-pwb.png')
local_filename = os.path.join(_data_dir, 'MP_sounds.png')
self.site.upload(page,
source_filename=local_filename,
comment='pywikibot test',
ignore_warnings=True)
def test_getFirstUploader(self):
"""Test getFirstUploader."""
page = pywikibot.FilePage(self.site, 'File:Albert Einstein.jpg')
first = page.getFirstUploader()
self.assertOneDeprecation()
self.assertEqual(first, ['Herbizid', '2011-03-18T10:04:48Z'])
self.assertIsInstance(first[0], unicode)
self.assertIsInstance(first[1], unicode)
def testOnBoth(self):
title = 'File:Pulsante spam.png'
commons = self.get_site('commons')
itwp = self.get_site('itwiki')
itwp_file = pywikibot.FilePage(itwp, title)
for using in itwp_file.usingPages():
self.assertIsInstance(using, pywikibot.Page)
commons_file = pywikibot.FilePage(commons, title)
self.assertTrue(itwp_file.fileUrl())
self.assertTrue(itwp_file.exists())
self.assertTrue(commons_file.exists())
self.assertFalse(itwp_file.fileIsShared())
self.assertTrue(commons_file.fileIsShared())
def run(self):
self.site.login()
with open(self.file) as f:
for line in f:
icon_id, icon_name = line.strip().split(" ",1)
page_name = icon_name+".png"
page = pywikibot.FilePage(self.site, page_name)
self.edit_page(page, self.iconDir+icon_id)
pywikibot.output('Changed '+str(self.edited)+' icons.')
def testOnBoth(self):
"""Test file_is_shared() on file page with local and shared file."""
title = 'File:Pulsante spam.png'
commons = self.get_site('commons')
itwp = self.get_site('itwiki')
itwp_file = pywikibot.FilePage(itwp, title)
for using in itwp_file.usingPages():
self.assertIsInstance(using, pywikibot.Page)
commons_file = pywikibot.FilePage(commons, title)
self.assertTrue(itwp_file.get_file_url())
self.assertTrue(itwp_file.exists())
self.assertTrue(commons_file.exists())
self.assertFalse(itwp_file.file_is_shared())
self.assertTrue(commons_file.file_is_shared())
def page(self):
"""
Return FilePage on which action was performed.
@rtype: pywikibot.FilePage
"""
if not hasattr(self, '_page'):
self._page = pywikibot.FilePage(self.site, self.data['title'])
return self._page
def main(timer):
commons = pwb.Site("commons", "commons")
while True:
category = make_cat(get_time(12))
pgs = get_cat_members(category)
pg = select_random(pgs).get("title")
pg = pwb.FilePage(commons, pg)
tweet_it(pg, get_time(24))
wait_it(timer)
def fitxer_existeix(nomfit):
commons = pywikibot.Site('commons', 'commons')
wp_file = pywikibot.FilePage(commons, nomfit)
try:
fitxerexisteix = wp_file.exists()
except:
fitxerexisteix = False
# Si ja l'hem trobat a Commons no cal buscar més
if fitxerexisteix:
return True
# ara busquem a cawiki
casite = pywikibot.Site('ca')
wp_file = pywikibot.FilePage(casite, nomfit)
try:
fitxerexisteix = wp_file.exists()
except:
fitxerexisteix = False
return fitxerexisteix
def upload_image(self, debug=False):
"""Upload the image at self.url to the target wiki.
Return the filename that was used to upload the image.
If the upload fails, ask the user whether to try again or not.
If the user chooses not to retry, return null.
"""
filename = self.process_filename()
site = self.targetSite
imagepage = pywikibot.FilePage(site, filename) # normalizes filename
imagepage.text = self.description
pywikibot.output(u'Uploading file to %s via API....' % site)
try:
if self.uploadByUrl:
site.upload(imagepage, source_url=self.url,
ignore_warnings=self.ignoreWarning)
else:
if "://" in self.url:
temp = self.read_file_content()
else:
temp = self.url
site.upload(imagepage, source_filename=temp,
ignore_warnings=self.ignoreWarning,
chunk_size=self.chunk_size)
except pywikibot.data.api.UploadWarning as warn:
pywikibot.output(
u'We got a warning message: {0}'.format(warn.message))
if self.abort_on_warn(warn.code):
answer = "N"
else:
answer = pywikibot.inputChoice(u"Do you want to ignore?",
['Yes', 'No'], ['y', 'N'], 'N')
if answer == "y":
self.ignoreWarning = 1
self.keepFilename = True
return self.upload_image(debug)
else:
pywikibot.output(u"Upload aborted.")
return
except pywikibot.data.api.APIError as error:
if error.code == u'uploaddisabled':
pywikibot.error("Upload error: Local file uploads are disabled on %s."
% site)
else:
pywikibot.error("Upload error: ", exc_info=True)
except Exception:
pywikibot.error("Upload error: ", exc_info=True)
else:
# No warning, upload complete.
pywikibot.output(u"Upload successful.")
return filename # data['filename']
def upload_file(self, file_url, debug=False, _file_key=None, _offset=0):
"""Upload the image at file_url to the target wiki.
Return the filename that was used to upload the image.
If the upload fails, ask the user whether to try again or not.
If the user chooses not to retry, return null.
"""
filename = self.process_filename(file_url)
if not filename:
return None
site = self.targetSite
imagepage = pywikibot.FilePage(site, filename) # normalizes filename
imagepage.text = self.description
pywikibot.output(u'Uploading file to %s via API...' % site)
success = False
try:
if self.ignoreWarning is True:
apiIgnoreWarnings = True
else:
apiIgnoreWarnings = self._handle_warnings
if self.uploadByUrl:
success = site.upload(imagepage, source_url=file_url,
ignore_warnings=apiIgnoreWarnings,
_file_key=_file_key, _offset=_offset)
else:
if "://" in file_url:
temp = self.read_file_content(file_url)
else:
temp = file_url
success = site.upload(imagepage, source_filename=temp,
ignore_warnings=apiIgnoreWarnings,
chunk_size=self.chunk_size,
_file_key=_file_key, _offset=_offset)
except pywikibot.data.api.APIError as error:
if error.code == u'uploaddisabled':
pywikibot.error("Upload error: Local file uploads are disabled on %s."
% site)
else:
pywikibot.error("Upload error: ", exc_info=True)
return None
except Exception:
pywikibot.error("Upload error: ", exc_info=True)
return None
else:
if success:
# No warning, upload complete.
pywikibot.output(u"Upload of %s successful." % filename)
return filename # data['filename']
else:
pywikibot.output(u"Upload aborted.")
return None
def format_claim_value(claim, value):
"""
Reformat the internal claim as the relevant pywikibot object.
@param claim: pywikibot.Claim to which value should be added
@param value: str|dict encoding the value to be added
@return: pywikibot representation of the claim value
"""
repo = claim.repo
if claim.type == 'wikibase-item':
return pywikibot.ItemPage(repo, value)
elif claim.type == 'commonsMedia':
return pywikibot.FilePage(_get_commons(), value)
elif claim.type == 'geo-shape':
return pywikibot.WbGeoShape(
pywikibot.Page(repo.geo_shape_repository(), value))
elif claim.type == 'tabular-data':
return pywikibot.WbTabularData(
pywikibot.Page(repo.tabular_data_repository(), value))
elif claim.type == 'monolingualtext':
if common.is_str(value):
text, _, lang = value.partition('@')
value = {'text': text, 'lang': lang}
return pywikibot.WbMonolingualText(value.get('text'),
value.get('lang'))
elif claim.type == 'globe-coordinate':
if common.is_str(value):
parts = value.replace(',', '@').split('@')
value = {parts[1]: parts[0], parts[3]: parts[2]}
# set precision to the least precise of the values
precision = max(coord_precision(value.get('lat')),
coord_precision(value.get('lon')))
return pywikibot.Coordinate(float(value.get('lat')),
float(value.get('lon')),
precision=precision)
elif claim.type == 'quantity':
if common.is_str(value):
amount, _, unit = value.partition('@')
value = {'amount': amount, 'unit': unit}
if value.get('unit'):
return pywikibot.WbQuantity(value.get('amount'),
pywikibot.ItemPage(
repo, value.get('unit')),
site=repo)
return pywikibot.WbQuantity(value.get('amount'), site=repo)
elif claim.type == 'time':
# note that Wikidata only supports precision down to day
# as a result pywikibot.WbTime.fromTimestr will produce an incompatible
# result for a fully qualified timestamp/timestr
return iso_to_wbtime(value)
# simple strings
return value
def getGenerator(self, fullrun):
"""
Get the generator to work on. Looks in the promotion date category. By default starts 14 days ago.
Yields dict with these fields:
* page
* mediaid
* starttime
"""
continuemore = True
cmcontinue = None
if fullrun:
cmstartsortkeyprefix = '0'
else:
cmstartsortkeyprefix = int(
(datetime.utcnow() + timedelta(days=-14)).timestamp())
while continuemore:
postdata = {
'action': 'query',
'format': 'json',
'list': 'categorymembers',
'cmtitle': 'Category:Valued_images_sorted_by_promotion_date',
'cmprop': 'ids|sortkey|sortkeyprefix|timestamp|title|type',
'cmtype': 'file',
'cmlimit': 100,
'cmcontinue': cmcontinue,
'cmstartsortkeyprefix': cmstartsortkeyprefix,
}
request = self.site._simple_request(**postdata)
data = request.submit()
if data.get('query-continue') and data.get('query-continue').get('categorymembers') \
and data.get('query-continue').get('categorymembers').get('cmcontinue'):
cmcontinue = data.get('query-continue').get(
'categorymembers').get('cmcontinue')
else:
cmcontinue = None
continuemore = False
for categorymember in data.get('query').get('categorymembers'):
try:
yield {
'page':
pywikibot.FilePage(self.site,
title=categorymember.get('title')),
'mediaid':
'M%s' % (categorymember.get('pageid')),
'starttime':
datetime.utcfromtimestamp(
int(categorymember.get('sortkeyprefix'))).strftime(
'%Y-%m-%d'),
}
except ValueError:
# Some dates are not correctly formatted
continue
def set_target(self, snak, value):
if value in ('somevalue', 'novalue'):
snak.setSnakType(value)
return True
if snak.type == 'wikibase-item':
snak.setTarget(pywikibot.ItemPage(self.repo, value))
return True
elif snak.type == 'wikibase-property':
snak.setTarget(pywikibot.PropertyPage(self.repo, value))
return True
elif snak.type == 'quantity':
match = self.quantityR.fullmatch(value)
if match:
amount, error, unit = match.groups()
else:
match = self.quantity_oldR.fullmatch(value)
if match:
amount, lower, upper, unit = match.groups()
error = upper, lower # it *is* the other way around
if match:
if unit:
unit = pywikibot.ItemPage(self.repo, 'Q' + unit)
quantity = WbQuantity(amount, unit, error, site=self.repo)
snak.setTarget(quantity)
return True
elif snak.type == 'time':
iso, _, prec = value.rpartition('/')
if iso:
time = WbTime.fromTimestr(iso,
precision=int(prec),
site=self.repo)
snak.setTarget(time)
return True
elif snak.type in ('string', 'external-id', 'url', 'math'):
if value.startswith('"') and value.endswith('"'):
snak.setTarget(value[1:-1])
return True
elif snak.type == 'commonsMedia':
if value.startswith('"') and value.endswith('"'):
repo = self.repo.image_repository()
snak.setTarget(pywikibot.FilePage(repo, value[1:-1]))
return True
#elif snak.type in ('geo-shape', 'tabular-data'):
elif snak.type == 'monolingualtext':
lang, _, text = value.partition(':')
if text and text.startswith('"') and text.endswith('"'):
monotext = WbMonolingualText(text[1:-1], lang)
snak.setTarget(monotext)
return True
elif snak.type == 'globe-coordinate':
match = self.globeR.fullmatch(value)
if match:
coord = Coordinate(*map(float, match.groups()), site=self.repo)
snak.setTarget(coord)
return True
return False
def get_exists_file(self, title):
file = pywikibot.FilePage(self.site, title)
if file.exists():
return file
try:
if file.file_is_shared():
return file
except Exception:
pass
return None
def action_normalize(item, job):
for claim in item.claims[job['p']]:
m = claim.toJSON()
curVal = m['mainsnak']['datavalue']['value']
newVal = curVal.replace('_', ' ')
if newVal[0:5] == 'File:':
newVal = newVal[5:]
target = pywikibot.FilePage(siteCommons, newVal)
if target.exists():
claim.changeTarget(target)
def test_file_info_with_no_page(self):
"""FilePage:latest_file_info raises NoPageError for missing pages."""
site = self.get_site()
image = pywikibot.FilePage(site, 'File:NoPage')
self.assertFalse(image.exists())
with self.assertRaisesRegex(
NoPageError, (r'Page \[\[(wikipedia\:|)test:File:NoPage\]\] '
r"doesn't exist\.")):
image = image.latest_file_info
def checkImageExists(title):
image = pywikibot.FilePage(site, title)
if image.exists():
return True
try:
if image.file_is_shared():
return True
except Exception:
pass
return False
def main():
data = {
"action": "query",
"format": "json",
"prop": "imageinfo|revisions",
"generator": "categorymembers",
"utf8": 1,
"iiprop": "user|url|timestamp",
"iilimit": "max",
"iilocalonly": 1,
"gcmtitle": tracker,
"gcmtype": "file",
"rvprop": "content",
"rvslots": "main",
"gcmlimit": "max"
}
c = True
while (c):
batch = r(bn, parameters=data).submit()
c = 'query-continue' in batch
if (c):
data['gcmcontinue'] = batch['query-continue']['categorymembers'][
'gcmcontinue']
batch = batch['query']['pages']
for i in batch:
i = batch[i]
name = i['title']
if name[-4:].lower() == '.svg':
#print("SVG found")
continue
infos = i['imageinfo']
if ((now - dt.strptime(infos[0]['timestamp'], ISO)).days < 7):
#print("7 days did not pass")
continue
rev = i['revisions'][0]['slots']['main']['*']
rev, n = furd_template.subn('', rev, 1)
if n is 0:
#print("Skip as not template was found")
continue
ids = []
for j in infos[1:]:
if 'url' in j:
k = archiveID.search(j['url'])
if (k):
ids.append(k.group(1))
if len(ids) is 0:
# No version was deleted
continue
try:
delete(name, ids)
i = pb.FilePage(bn, name)
i.text = rev
i.save(summary)
except Exception as e:
print("Something occurred:%s" % e)
def test_not_existing_download(self):
"""Test not existing download."""
page = pywikibot.FilePage(self.site,
'File:Albert Einstein.jpg_notexisting')
filename = join_images_path('Albert Einstein.jpg')
with self.assertRaisesRegex(
NoPageError,
re.escape('Page [[commons:File:Albert Einstein.jpg '
"notexisting]] doesn't exist.")):
page.download(filename)
def test_file_info_with_no_file(self):
"""FilePage:latest_file_info raises PagerelatedError if no file."""
site = self.get_site()
image = pywikibot.FilePage(site, 'File:Test with no image')
self.assertTrue(image.exists())
with self.assertRaisesRegex(
PageRelatedError,
(r'loadimageinfo: Query on '
r'\[\[(wikipedia\:|)test:File:Test with no image\]\]'
r' returned no imageinfo')):
image = image.latest_file_info
def test_file_info_with_no_page(self):
"""FilePage:latest_file_info raises NoPage for non existing pages."""
site = self.get_site()
image = pywikibot.FilePage(site, u'File:NoPage')
self.assertFalse(image.exists())
with self.assertRaisesRegex(
pywikibot.NoPage,
(r'Page \[\[(wikipedia\:|)test:File:NoPage\]\] '
r'doesn\'t exist\.')):
image = image.latest_file_info
def testSharedOnly(self):
title = 'File:Sepp Maier 1.JPG'
commons = self.get_site('commons')
itwp = self.get_site('itwiki')
itwp_file = pywikibot.FilePage(itwp, title)
for using in itwp_file.usingPages():
self.assertIsInstance(using, pywikibot.Page)
commons_file = pywikibot.FilePage(commons, title)
self.assertFalse(itwp_file.exists())
self.assertTrue(commons_file.exists())
self.assertTrue(itwp_file.fileIsShared())
self.assertTrue(commons_file.fileIsShared())
self.assertTrue(commons_file.fileUrl())
self.assertIn('/wikipedia/commons/', itwp_file.fileUrl())
self.assertRaises(pywikibot.NoPage, itwp_file.get)
def testLocalOnly(self):
title = 'File:April Fools Day Adminship discussion (2005).png'
commons = self.get_site('commons')
enwp = self.get_site('enwiki')
enwp_file = pywikibot.FilePage(enwp, title)
for using in enwp_file.usingPages():
self.assertIsInstance(using, pywikibot.Page)
commons_file = pywikibot.FilePage(commons, title)
self.assertTrue(enwp_file.fileUrl())
self.assertTrue(enwp_file.exists())
self.assertFalse(commons_file.exists())
self.assertFalse(enwp_file.fileIsShared())
self.assertRaises(pywikibot.NoPage, commons_file.fileIsShared)
self.assertRaises(pywikibot.NoPage, commons_file.fileUrl)
self.assertRaises(pywikibot.NoPage, commons_file.get)
def run(self):
"""Starts the bot."""
for page in self.generator:
self.current_page = page
item = pywikibot.ItemPage.fromPage(page)
if item.exists():
pywikibot.output(u'Found %s' % item.title())
imagename = page.properties().get('page_image')
if imagename:
claims = item.get().get('claims')
if self.wdproperty in claims:
pywikibot.output(
u'Item %s already contains image (%s)' %
(item.title(), self.wdproperty))
else:
newclaim = pywikibot.Claim(self.repo, self.wdproperty)
commonssite = pywikibot.Site("commons", "commons")
imagelink = pywikibot.Link(imagename,
source=commonssite,
defaultNamespace=6)
image = pywikibot.FilePage(imagelink)
if image.isRedirectPage():
image = pywikibot.FilePage(
image.getRedirectTarget())
if not image.exists():
pywikibot.output(
'[[%s]] doesn\'t exist so I can\'t link to it'
% (image.title(), ))
continue
newclaim.setTarget(image)
pywikibot.output(
'Adding %s --> %s' %
(newclaim.getID(), newclaim.getTarget()))
item.addClaim(newclaim)
# A generator might yield pages from multiple sites
source = self.getSource(page.site)
if source:
newclaim.addSource(source, bot=True)
def test_get_file_history_cache(self):
"""Test the cache mechanism of get_file_history."""
with mock.patch.object(self.site, 'loadimageinfo', autospec=True):
page = pywikibot.FilePage(self.site, 'File:Foo.jpg')
_file_revisions = page.get_file_history()
# On the first call the history is loaded via API
self.assertEqual(_file_revisions, {})
# Fill the cache
_file_revisions['foo'] = 'bar'
# On the second call page._file_revisions is returned
self.assertEqual(page.get_file_history(), {'foo': 'bar'})
self.site.loadimageinfo.assert_called_once_with(page, history=True)
def run(site, image, isCorrupt, date_scanned, to_delete_nom):
image_page = pywikibot.FilePage(site, image)
_, ext = os.path.splitext(image_page.title()) # get filetype
if not icu.allow_bots(image_page.text, "TheSandBot"):
print("Not to edit " + image_page.title())
return
download_attempts = 0
failed = 0
while True:
#with open("./Example4" + ext, "wb") as fd:
image_page.download("./Example4" + ext)
hash_result, img_hash = icu.verify_hash(site, "./Example4" + ext, image_page)
if not hash_result:
if download_attempts >= 10:
failed = 1
break
download_attempts += 1
continue
else:
break
if failed:
raise ValueError(
"Hash check failed for ./Example4{0} vs {1} {2} times. Aborting...".format(ext, str(image_page.title()),
str(download_attempts)))
del download_attempts
try:
result = icu.image_is_corrupt("./Example4" + ext)
except UnidentifiedImageError:
os.remove("./Example4" + ext) # file not an image.
raise
del ext # no longer a needed variable
if result: # image corrupt
return
else: # image not corrupt
edit_summary = "Removing [[Template:TSB image identified corrupt]] - image no longer corrupt"
code = mwparserfromhell.parse(image_page.text)
for template in code.filter_templates():
if template.name.matches("TSB image identified corrupt"):
code.remove(template) # template no longer needed
try:
pwb_wrappers.retry_apierror(
lambda:
image_page.save(text=str(code),
summary=edit_summary, minor=False, botflag=True, force=True)
)
except pywikibot.exceptions.LockedPage as e:
print(image_page.title())
print(e.message)
# update database entry to set image as no longer corrupt and nullify to_delete_nom
db.update_entry(str(image_page.title()), False, None, img_hash, was_fixed=True)
