def test_getLatestUploader(self): """Test getLatestUploader.""" page = pywikibot.FilePage(self.site, 'File:Albert Einstein.jpg') latest = page.getLatestUploader() self.assertOneDeprecation() self.assertEqual(len(latest), 2) self.assertIsInstance(latest[0], unicode) self.assertIsInstance(latest[1], unicode)
def _finish_upload(self, chunk_size, file_name): """Finish the upload.""" # Finish/continue upload with the given file key page = pywikibot.FilePage(self.site, 'MP_sounds-pwb.png') self.site.upload(page, source_filename=file_name, comment='pywikibot test', chunk_size=chunk_size, ignore_warnings=True, report_success=False, _file_key=self._file_key, _offset=self._offset)
def test_globalusage(self, key): """Test globalusage generator.""" page = pywikibot.FilePage(self.site, 'File:Example.jpg') gen = page.globalusage(total=3) pages = list(gen) self.assertEqual(len(pages), 3) self.assertTrue(any(isinstance(p), pywikibot.Page) for p in pages) self.assertTrue(any(p.site != self.site for p in pages))
def test_png_chunked(self): """Test uploading a png in two chunks using Site.upload.""" page = pywikibot.FilePage(self.site, 'MP_sounds-pwb-chunked.png') self.site.upload(page, source_filename=self.sounds_png, comment='pywikibot test', ignore_warnings=True, chunk_size=1024)
def test_png(self): """Test uploading a png using Site.upload.""" page = pywikibot.FilePage(self.site, 'MP_sounds-pwb.png') local_filename = os.path.join(_data_dir, 'MP_sounds.png') self.site.upload(page, source_filename=local_filename, comment='pywikibot test', ignore_warnings=True)
def test_getFirstUploader(self): """Test getFirstUploader.""" page = pywikibot.FilePage(self.site, 'File:Albert Einstein.jpg') first = page.getFirstUploader() self.assertOneDeprecation() self.assertEqual(first, ['Herbizid', '2011-03-18T10:04:48Z']) self.assertIsInstance(first[0], unicode) self.assertIsInstance(first[1], unicode)
def testOnBoth(self): title = 'File:Pulsante spam.png' commons = self.get_site('commons') itwp = self.get_site('itwiki') itwp_file = pywikibot.FilePage(itwp, title) for using in itwp_file.usingPages(): self.assertIsInstance(using, pywikibot.Page) commons_file = pywikibot.FilePage(commons, title) self.assertTrue(itwp_file.fileUrl()) self.assertTrue(itwp_file.exists()) self.assertTrue(commons_file.exists()) self.assertFalse(itwp_file.fileIsShared()) self.assertTrue(commons_file.fileIsShared())
def run(self): self.site.login() with open(self.file) as f: for line in f: icon_id, icon_name = line.strip().split(" ",1) page_name = icon_name+".png" page = pywikibot.FilePage(self.site, page_name) self.edit_page(page, self.iconDir+icon_id) pywikibot.output('Changed '+str(self.edited)+' icons.')
def testOnBoth(self): """Test file_is_shared() on file page with local and shared file.""" title = 'File:Pulsante spam.png' commons = self.get_site('commons') itwp = self.get_site('itwiki') itwp_file = pywikibot.FilePage(itwp, title) for using in itwp_file.usingPages(): self.assertIsInstance(using, pywikibot.Page) commons_file = pywikibot.FilePage(commons, title) self.assertTrue(itwp_file.get_file_url()) self.assertTrue(itwp_file.exists()) self.assertTrue(commons_file.exists()) self.assertFalse(itwp_file.file_is_shared()) self.assertTrue(commons_file.file_is_shared())
def page(self): """ Return FilePage on which action was performed. @rtype: pywikibot.FilePage """ if not hasattr(self, '_page'): self._page = pywikibot.FilePage(self.site, self.data['title']) return self._page
def main(timer): commons = pwb.Site("commons", "commons") while True: category = make_cat(get_time(12)) pgs = get_cat_members(category) pg = select_random(pgs).get("title") pg = pwb.FilePage(commons, pg) tweet_it(pg, get_time(24)) wait_it(timer)
def fitxer_existeix(nomfit): commons = pywikibot.Site('commons', 'commons') wp_file = pywikibot.FilePage(commons, nomfit) try: fitxerexisteix = wp_file.exists() except: fitxerexisteix = False # Si ja l'hem trobat a Commons no cal buscar més if fitxerexisteix: return True # ara busquem a cawiki casite = pywikibot.Site('ca') wp_file = pywikibot.FilePage(casite, nomfit) try: fitxerexisteix = wp_file.exists() except: fitxerexisteix = False return fitxerexisteix
def upload_image(self, debug=False): """Upload the image at self.url to the target wiki. Return the filename that was used to upload the image. If the upload fails, ask the user whether to try again or not. If the user chooses not to retry, return null. """ filename = self.process_filename() site = self.targetSite imagepage = pywikibot.FilePage(site, filename) # normalizes filename imagepage.text = self.description pywikibot.output(u'Uploading file to %s via API....' % site) try: if self.uploadByUrl: site.upload(imagepage, source_url=self.url, ignore_warnings=self.ignoreWarning) else: if "://" in self.url: temp = self.read_file_content() else: temp = self.url site.upload(imagepage, source_filename=temp, ignore_warnings=self.ignoreWarning, chunk_size=self.chunk_size) except pywikibot.data.api.UploadWarning as warn: pywikibot.output( u'We got a warning message: {0}'.format(warn.message)) if self.abort_on_warn(warn.code): answer = "N" else: answer = pywikibot.inputChoice(u"Do you want to ignore?", ['Yes', 'No'], ['y', 'N'], 'N') if answer == "y": self.ignoreWarning = 1 self.keepFilename = True return self.upload_image(debug) else: pywikibot.output(u"Upload aborted.") return except pywikibot.data.api.APIError as error: if error.code == u'uploaddisabled': pywikibot.error("Upload error: Local file uploads are disabled on %s." % site) else: pywikibot.error("Upload error: ", exc_info=True) except Exception: pywikibot.error("Upload error: ", exc_info=True) else: # No warning, upload complete. pywikibot.output(u"Upload successful.") return filename # data['filename']
def upload_file(self, file_url, debug=False, _file_key=None, _offset=0): """Upload the image at file_url to the target wiki. Return the filename that was used to upload the image. If the upload fails, ask the user whether to try again or not. If the user chooses not to retry, return null. """ filename = self.process_filename(file_url) if not filename: return None site = self.targetSite imagepage = pywikibot.FilePage(site, filename) # normalizes filename imagepage.text = self.description pywikibot.output(u'Uploading file to %s via API...' % site) success = False try: if self.ignoreWarning is True: apiIgnoreWarnings = True else: apiIgnoreWarnings = self._handle_warnings if self.uploadByUrl: success = site.upload(imagepage, source_url=file_url, ignore_warnings=apiIgnoreWarnings, _file_key=_file_key, _offset=_offset) else: if "://" in file_url: temp = self.read_file_content(file_url) else: temp = file_url success = site.upload(imagepage, source_filename=temp, ignore_warnings=apiIgnoreWarnings, chunk_size=self.chunk_size, _file_key=_file_key, _offset=_offset) except pywikibot.data.api.APIError as error: if error.code == u'uploaddisabled': pywikibot.error("Upload error: Local file uploads are disabled on %s." % site) else: pywikibot.error("Upload error: ", exc_info=True) return None except Exception: pywikibot.error("Upload error: ", exc_info=True) return None else: if success: # No warning, upload complete. pywikibot.output(u"Upload of %s successful." % filename) return filename # data['filename'] else: pywikibot.output(u"Upload aborted.") return None
def format_claim_value(claim, value): """ Reformat the internal claim as the relevant pywikibot object. @param claim: pywikibot.Claim to which value should be added @param value: str|dict encoding the value to be added @return: pywikibot representation of the claim value """ repo = claim.repo if claim.type == 'wikibase-item': return pywikibot.ItemPage(repo, value) elif claim.type == 'commonsMedia': return pywikibot.FilePage(_get_commons(), value) elif claim.type == 'geo-shape': return pywikibot.WbGeoShape( pywikibot.Page(repo.geo_shape_repository(), value)) elif claim.type == 'tabular-data': return pywikibot.WbTabularData( pywikibot.Page(repo.tabular_data_repository(), value)) elif claim.type == 'monolingualtext': if common.is_str(value): text, _, lang = value.partition('@') value = {'text': text, 'lang': lang} return pywikibot.WbMonolingualText(value.get('text'), value.get('lang')) elif claim.type == 'globe-coordinate': if common.is_str(value): parts = value.replace(',', '@').split('@') value = {parts[1]: parts[0], parts[3]: parts[2]} # set precision to the least precise of the values precision = max(coord_precision(value.get('lat')), coord_precision(value.get('lon'))) return pywikibot.Coordinate(float(value.get('lat')), float(value.get('lon')), precision=precision) elif claim.type == 'quantity': if common.is_str(value): amount, _, unit = value.partition('@') value = {'amount': amount, 'unit': unit} if value.get('unit'): return pywikibot.WbQuantity(value.get('amount'), pywikibot.ItemPage( repo, value.get('unit')), site=repo) return pywikibot.WbQuantity(value.get('amount'), site=repo) elif claim.type == 'time': # note that Wikidata only supports precision down to day # as a result pywikibot.WbTime.fromTimestr will produce an incompatible # result for a fully qualified timestamp/timestr return iso_to_wbtime(value) # simple strings return value
def getGenerator(self, fullrun): """ Get the generator to work on. Looks in the promotion date category. By default starts 14 days ago. Yields dict with these fields: * page * mediaid * starttime """ continuemore = True cmcontinue = None if fullrun: cmstartsortkeyprefix = '0' else: cmstartsortkeyprefix = int( (datetime.utcnow() + timedelta(days=-14)).timestamp()) while continuemore: postdata = { 'action': 'query', 'format': 'json', 'list': 'categorymembers', 'cmtitle': 'Category:Valued_images_sorted_by_promotion_date', 'cmprop': 'ids|sortkey|sortkeyprefix|timestamp|title|type', 'cmtype': 'file', 'cmlimit': 100, 'cmcontinue': cmcontinue, 'cmstartsortkeyprefix': cmstartsortkeyprefix, } request = self.site._simple_request(**postdata) data = request.submit() if data.get('query-continue') and data.get('query-continue').get('categorymembers') \ and data.get('query-continue').get('categorymembers').get('cmcontinue'): cmcontinue = data.get('query-continue').get( 'categorymembers').get('cmcontinue') else: cmcontinue = None continuemore = False for categorymember in data.get('query').get('categorymembers'): try: yield { 'page': pywikibot.FilePage(self.site, title=categorymember.get('title')), 'mediaid': 'M%s' % (categorymember.get('pageid')), 'starttime': datetime.utcfromtimestamp( int(categorymember.get('sortkeyprefix'))).strftime( '%Y-%m-%d'), } except ValueError: # Some dates are not correctly formatted continue
def set_target(self, snak, value): if value in ('somevalue', 'novalue'): snak.setSnakType(value) return True if snak.type == 'wikibase-item': snak.setTarget(pywikibot.ItemPage(self.repo, value)) return True elif snak.type == 'wikibase-property': snak.setTarget(pywikibot.PropertyPage(self.repo, value)) return True elif snak.type == 'quantity': match = self.quantityR.fullmatch(value) if match: amount, error, unit = match.groups() else: match = self.quantity_oldR.fullmatch(value) if match: amount, lower, upper, unit = match.groups() error = upper, lower # it *is* the other way around if match: if unit: unit = pywikibot.ItemPage(self.repo, 'Q' + unit) quantity = WbQuantity(amount, unit, error, site=self.repo) snak.setTarget(quantity) return True elif snak.type == 'time': iso, _, prec = value.rpartition('/') if iso: time = WbTime.fromTimestr(iso, precision=int(prec), site=self.repo) snak.setTarget(time) return True elif snak.type in ('string', 'external-id', 'url', 'math'): if value.startswith('"') and value.endswith('"'): snak.setTarget(value[1:-1]) return True elif snak.type == 'commonsMedia': if value.startswith('"') and value.endswith('"'): repo = self.repo.image_repository() snak.setTarget(pywikibot.FilePage(repo, value[1:-1])) return True #elif snak.type in ('geo-shape', 'tabular-data'): elif snak.type == 'monolingualtext': lang, _, text = value.partition(':') if text and text.startswith('"') and text.endswith('"'): monotext = WbMonolingualText(text[1:-1], lang) snak.setTarget(monotext) return True elif snak.type == 'globe-coordinate': match = self.globeR.fullmatch(value) if match: coord = Coordinate(*map(float, match.groups()), site=self.repo) snak.setTarget(coord) return True return False
def get_exists_file(self, title): file = pywikibot.FilePage(self.site, title) if file.exists(): return file try: if file.file_is_shared(): return file except Exception: pass return None
def action_normalize(item, job): for claim in item.claims[job['p']]: m = claim.toJSON() curVal = m['mainsnak']['datavalue']['value'] newVal = curVal.replace('_', ' ') if newVal[0:5] == 'File:': newVal = newVal[5:] target = pywikibot.FilePage(siteCommons, newVal) if target.exists(): claim.changeTarget(target)
def test_file_info_with_no_page(self): """FilePage:latest_file_info raises NoPageError for missing pages.""" site = self.get_site() image = pywikibot.FilePage(site, 'File:NoPage') self.assertFalse(image.exists()) with self.assertRaisesRegex( NoPageError, (r'Page \[\[(wikipedia\:|)test:File:NoPage\]\] ' r"doesn't exist\.")): image = image.latest_file_info
def checkImageExists(title): image = pywikibot.FilePage(site, title) if image.exists(): return True try: if image.file_is_shared(): return True except Exception: pass return False
def main(): data = { "action": "query", "format": "json", "prop": "imageinfo|revisions", "generator": "categorymembers", "utf8": 1, "iiprop": "user|url|timestamp", "iilimit": "max", "iilocalonly": 1, "gcmtitle": tracker, "gcmtype": "file", "rvprop": "content", "rvslots": "main", "gcmlimit": "max" } c = True while (c): batch = r(bn, parameters=data).submit() c = 'query-continue' in batch if (c): data['gcmcontinue'] = batch['query-continue']['categorymembers'][ 'gcmcontinue'] batch = batch['query']['pages'] for i in batch: i = batch[i] name = i['title'] if name[-4:].lower() == '.svg': #print("SVG found") continue infos = i['imageinfo'] if ((now - dt.strptime(infos[0]['timestamp'], ISO)).days < 7): #print("7 days did not pass") continue rev = i['revisions'][0]['slots']['main']['*'] rev, n = furd_template.subn('', rev, 1) if n is 0: #print("Skip as not template was found") continue ids = [] for j in infos[1:]: if 'url' in j: k = archiveID.search(j['url']) if (k): ids.append(k.group(1)) if len(ids) is 0: # No version was deleted continue try: delete(name, ids) i = pb.FilePage(bn, name) i.text = rev i.save(summary) except Exception as e: print("Something occurred:%s" % e)
def test_not_existing_download(self): """Test not existing download.""" page = pywikibot.FilePage(self.site, 'File:Albert Einstein.jpg_notexisting') filename = join_images_path('Albert Einstein.jpg') with self.assertRaisesRegex( NoPageError, re.escape('Page [[commons:File:Albert Einstein.jpg ' "notexisting]] doesn't exist.")): page.download(filename)
def test_file_info_with_no_file(self): """FilePage:latest_file_info raises PagerelatedError if no file.""" site = self.get_site() image = pywikibot.FilePage(site, 'File:Test with no image') self.assertTrue(image.exists()) with self.assertRaisesRegex( PageRelatedError, (r'loadimageinfo: Query on ' r'\[\[(wikipedia\:|)test:File:Test with no image\]\]' r' returned no imageinfo')): image = image.latest_file_info
def test_file_info_with_no_page(self): """FilePage:latest_file_info raises NoPage for non existing pages.""" site = self.get_site() image = pywikibot.FilePage(site, u'File:NoPage') self.assertFalse(image.exists()) with self.assertRaisesRegex( pywikibot.NoPage, (r'Page \[\[(wikipedia\:|)test:File:NoPage\]\] ' r'doesn\'t exist\.')): image = image.latest_file_info
def testSharedOnly(self): title = 'File:Sepp Maier 1.JPG' commons = self.get_site('commons') itwp = self.get_site('itwiki') itwp_file = pywikibot.FilePage(itwp, title) for using in itwp_file.usingPages(): self.assertIsInstance(using, pywikibot.Page) commons_file = pywikibot.FilePage(commons, title) self.assertFalse(itwp_file.exists()) self.assertTrue(commons_file.exists()) self.assertTrue(itwp_file.fileIsShared()) self.assertTrue(commons_file.fileIsShared()) self.assertTrue(commons_file.fileUrl()) self.assertIn('/wikipedia/commons/', itwp_file.fileUrl()) self.assertRaises(pywikibot.NoPage, itwp_file.get)
def testLocalOnly(self): title = 'File:April Fools Day Adminship discussion (2005).png' commons = self.get_site('commons') enwp = self.get_site('enwiki') enwp_file = pywikibot.FilePage(enwp, title) for using in enwp_file.usingPages(): self.assertIsInstance(using, pywikibot.Page) commons_file = pywikibot.FilePage(commons, title) self.assertTrue(enwp_file.fileUrl()) self.assertTrue(enwp_file.exists()) self.assertFalse(commons_file.exists()) self.assertFalse(enwp_file.fileIsShared()) self.assertRaises(pywikibot.NoPage, commons_file.fileIsShared) self.assertRaises(pywikibot.NoPage, commons_file.fileUrl) self.assertRaises(pywikibot.NoPage, commons_file.get)
def run(self): """Starts the bot.""" for page in self.generator: self.current_page = page item = pywikibot.ItemPage.fromPage(page) if item.exists(): pywikibot.output(u'Found %s' % item.title()) imagename = page.properties().get('page_image') if imagename: claims = item.get().get('claims') if self.wdproperty in claims: pywikibot.output( u'Item %s already contains image (%s)' % (item.title(), self.wdproperty)) else: newclaim = pywikibot.Claim(self.repo, self.wdproperty) commonssite = pywikibot.Site("commons", "commons") imagelink = pywikibot.Link(imagename, source=commonssite, defaultNamespace=6) image = pywikibot.FilePage(imagelink) if image.isRedirectPage(): image = pywikibot.FilePage( image.getRedirectTarget()) if not image.exists(): pywikibot.output( '[[%s]] doesn\'t exist so I can\'t link to it' % (image.title(), )) continue newclaim.setTarget(image) pywikibot.output( 'Adding %s --> %s' % (newclaim.getID(), newclaim.getTarget())) item.addClaim(newclaim) # A generator might yield pages from multiple sites source = self.getSource(page.site) if source: newclaim.addSource(source, bot=True)
def test_get_file_history_cache(self): """Test the cache mechanism of get_file_history.""" with mock.patch.object(self.site, 'loadimageinfo', autospec=True): page = pywikibot.FilePage(self.site, 'File:Foo.jpg') _file_revisions = page.get_file_history() # On the first call the history is loaded via API self.assertEqual(_file_revisions, {}) # Fill the cache _file_revisions['foo'] = 'bar' # On the second call page._file_revisions is returned self.assertEqual(page.get_file_history(), {'foo': 'bar'}) self.site.loadimageinfo.assert_called_once_with(page, history=True)
def run(site, image, isCorrupt, date_scanned, to_delete_nom): image_page = pywikibot.FilePage(site, image) _, ext = os.path.splitext(image_page.title()) # get filetype if not icu.allow_bots(image_page.text, "TheSandBot"): print("Not to edit " + image_page.title()) return download_attempts = 0 failed = 0 while True: #with open("./Example4" + ext, "wb") as fd: image_page.download("./Example4" + ext) hash_result, img_hash = icu.verify_hash(site, "./Example4" + ext, image_page) if not hash_result: if download_attempts >= 10: failed = 1 break download_attempts += 1 continue else: break if failed: raise ValueError( "Hash check failed for ./Example4{0} vs {1} {2} times. Aborting...".format(ext, str(image_page.title()), str(download_attempts))) del download_attempts try: result = icu.image_is_corrupt("./Example4" + ext) except UnidentifiedImageError: os.remove("./Example4" + ext) # file not an image. raise del ext # no longer a needed variable if result: # image corrupt return else: # image not corrupt edit_summary = "Removing [[Template:TSB image identified corrupt]] - image no longer corrupt" code = mwparserfromhell.parse(image_page.text) for template in code.filter_templates(): if template.name.matches("TSB image identified corrupt"): code.remove(template) # template no longer needed try: pwb_wrappers.retry_apierror( lambda: image_page.save(text=str(code), summary=edit_summary, minor=False, botflag=True, force=True) ) except pywikibot.exceptions.LockedPage as e: print(image_page.title()) print(e.message) # update database entry to set image as no longer corrupt and nullify to_delete_nom db.update_entry(str(image_page.title()), False, None, img_hash, was_fixed=True)