def main(give_url, image_url, desc): """Run the bot.""" url = give_url image_url = '' if url == '': if image_url: url = pywikibot.input(u"What URL range should I check " u"(use $ for the part that is changeable)") else: url = pywikibot.input(u"From what URL should I get the images?") if image_url: minimum = 1 maximum = 99 answer = pywikibot.input( u"What is the first number to check (default: 1)") if answer: minimum = int(answer) answer = pywikibot.input( u"What is the last number to check (default: 99)") if answer: maximum = int(answer) if not desc: basicdesc = pywikibot.input( u"What text should be added at the end of " u"the description of each image from this url?") else: basicdesc = desc if image_url: ilinks = [] i = minimum while i <= maximum: ilinks += [url.replace("$", str(i))] i += 1 else: ilinks = get_imagelinks(url) for image in ilinks: if pywikibot.input_yn('Include image %s?' % image, default=False, automatic_quit=False): desc = pywikibot.input(u"Give the description of this image:") categories = [] while True: cat = pywikibot.input(u"Specify a category (or press enter to " u"end adding categories)") if not cat.strip(): break if ":" in cat: categories.append(u"[[%s]]" % cat) else: categories.append(u"[[%s:%s]]" % (mysite.namespace(14), cat)) desc += "\r\n\r\n" + basicdesc + "\r\n\r\n" + \ "\r\n".join(categories) uploadBot = UploadRobot(image, description=desc) uploadBot.run() elif answer == 's': break
def processImage(self, fields): """Work on a single image.""" cid = self.buildNewImageDescription(fields) pywikibot.output(cid) bot = UploadRobot(url=fields.get('imagepage').fileUrl(), description=cid, useFilename=fields.get('filename'), keepFilename=True, verifyDescription=False, ignoreWarning=True, targetSite=pywikibot.Site('commons', 'commons')) bot.run() self.tagNowcommons(fields.get('imagepage'), fields.get('filename')) self.replaceUsage(fields.get('imagepage'), fields.get('filename'))
def processPhoto(photoInfo, panoramioreview=False, reviewer='', override=u'', addCategory=u'', autonomous=False, site=None): """Process a single Panoramio photo.""" if not site: site = pywikibot.Site('commons', 'commons') if isAllowedLicense(photoInfo) or override: # Should download the photo only once photo = downloadPhoto(photoInfo.get(u'photo_file_url')) # Don't upload duplicate images, should add override option duplicates = findDuplicateImages(photo, site=site) if duplicates: pywikibot.output(u'Found duplicate image at %s' % duplicates.pop()) else: filename = getFilename(photoInfo, site=site) pywikibot.output(filename) description = getDescription(photoInfo, panoramioreview, reviewer, override, addCategory) pywikibot.output(description) if not autonomous: (newDescription, newFilename, skip) = Tkdialog( description, photo, filename).show_dialog() else: newDescription = description newFilename = filename skip = False # pywikibot.output(newPhotoDescription) # if (pywikibot.Page(title=u'File:'+ filename, # site=pywikibot.Site()).exists()): # # I should probably check if the hash is the same and if not upload # # it under a different name # pywikibot.output(u'File:' + filename + u' already exists!') # else: # Do the actual upload # Would be nice to check before I upload if the file is already at # Commons # Not that important for this program, but maybe for derived # programs if not skip: bot = UploadRobot(photoInfo.get(u'photo_file_url'), description=newDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False, site=site) bot.upload_image(debug=False) return 1 return 0
def treat(self, photo): """Process each page.""" duplicates = photo.findDuplicateImages() if duplicates: pywikibot.output(u"Skipping duplicate of %r" % duplicates) return duplicates[0] title = photo.getTitle(self.titlefmt) description = photo.getDescription(self.pagefmt) bot = UploadRobot(url=photo.URL, description=description, useFilename=title, keepFilename=True, verifyDescription=False, targetSite=self.site) bot._contents = photo.downloadPhoto().getvalue() bot._retrieved = True bot.run() return title
def test_png(self): """Test uploading a png using upload.py.""" bot = UploadRobot(url=[join_images_path('MP_sounds.png')], target_site=self.get_site(), **self.params) bot.run()
def processPhoto(flickr, photo_id='', flickrreview=False, reviewer='', override='', addCategory='', removeCategories=False, autonomous=False): """Process a single Flickr photo. For each image: * Check the license * Check if it isn't already on Commons * Build suggested filename * Check for name collision and maybe alter it * Pull description from Flinfo * Show image and description to user * Add a nice hotcat lookalike for the adding of categories * Filter the categories * Upload the image """ if photo_id: pywikibot.output(str(photo_id)) (photoInfo, photoSizes) = getPhoto(flickr, photo_id) if isAllowedLicense(photoInfo) or override: # Get the url of the largest photo photoUrl = getPhotoUrl(photoSizes) # Should download the photo only once photo = downloadPhoto(photoUrl) # Don't upload duplicate images, should add override option duplicates = findDuplicateImages(photo) if duplicates: pywikibot.output('Found duplicate image at {}' .format(duplicates.pop())) else: filename = getFilename(photoInfo, photo_url=photoUrl) flinfoDescription = getFlinfoDescription(photo_id) if 'Blacklisted user' in flinfoDescription: pywikibot.warning('Blacklisted user found:\n' + flinfoDescription) return 0 photoDescription = buildDescription(flinfoDescription, flickrreview, reviewer, override, addCategory, removeCategories) # pywikibot.output(photoDescription) if not isinstance(Tkdialog, ImportError) and not autonomous: try: (newPhotoDescription, newFilename, skip) = Tkdialog( photoDescription, photo, filename).show_dialog() except ImportError as e: pywikibot.warning(e) pywikibot.warning('Switching to autonomous mode.') autonomous = True elif not autonomous: pywikibot.warning('Switching to autonomous mode because GUI ' 'interface cannot be used') pywikibot.warning(Tkdialog) autonomous = True if autonomous: newPhotoDescription = photoDescription newFilename = filename skip = False # Do the actual upload # Would be nice to check before I upload if the file is already at # Commons. Not that important for this program, but maybe for # derived programs if not skip: bot = UploadRobot(photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) bot.upload_image(debug=False) return 1 else: pywikibot.output('Invalid license') return 0
def run(self): """Run the bot.""" tosend = { 'language': self.imagePage.site.lang.encode('utf-8'), 'image': self.imagePage.title(with_ns=False).encode('utf-8'), 'newname': self.newname.encode('utf-8'), 'project': self.imagePage.site.family.name.encode('utf-8'), 'username': '', 'commonsense': '1', 'remove_categories': '1', 'ignorewarnings': '1', 'doit': 'Uitvoeren' } pywikibot.output(tosend) CH = pageTextPost('http://tools.wmflabs.org/commonshelper/index.php', tosend) pywikibot.output('Got CH desc.') tablock = CH.split('<textarea ')[1].split('>')[0] CH = CH.split('<textarea ' + tablock + '>')[1].split('</textarea>')[0] CH = CH.replace(u'×', u'×') CH = self.fixAuthor(CH) pywikibot.output(CH) # I want every picture to be tagged with the bottemplate so i can check # my contributions later. CH = ('\n\n{{BotMoveToCommons|' + self.imagePage.site.lang + '.' + self.imagePage.site.family.name + '|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}' '|day={{subst:CURRENTDAY}}}}' + CH) if self.category: CH = CH.replace( '{{subst:Unc}} <!-- Remove this line once you have ' 'added categories -->', '') CH += u'[[Category:' + self.category + u']]' bot = UploadRobot(url=self.imagePage.fileUrl(), description=CH, useFilename=self.newname, keepFilename=True, verifyDescription=False, ignoreWarning=True, targetSite=self.image_repo) bot.run() # Should check if the image actually was uploaded if pywikibot.Page(self.image_repo, u'Image:' + self.newname).exists(): # Get a fresh copy, force to get the page so we dont run into edit # conflicts imtxt = self.imagePage.get(force=True) # Remove the move to commons templates if self.imagePage.site.lang in moveToCommonsTemplate: for moveTemplate in moveToCommonsTemplate[ self.imagePage.site.lang]: imtxt = re.sub(r'(?i)\{\{' + moveTemplate + r'[^\}]*\}\}', '', imtxt) # add {{NowCommons}} if self.imagePage.site.lang in nowCommonsTemplate: addTemplate = nowCommonsTemplate[ self.imagePage.site.lang] % self.newname else: addTemplate = nowCommonsTemplate['_default'] % self.newname commentText = i18n.twtranslate( self.imagePage.site, 'commons-file-now-available', { 'localfile': self.imagePage.title(with_ns=False), 'commonsfile': self.newname }) pywikibot.showDiff(self.imagePage.get(), imtxt + addTemplate) self.imagePage.put(imtxt + addTemplate, comment=commentText) self.gen = pagegenerators.FileLinksGenerator(self.imagePage) self.preloadingGen = pagegenerators.PreloadingGenerator(self.gen) moveSummary = i18n.twtranslate( self.imagePage.site, 'commons-file-moved', { 'localfile': self.imagePage.title(with_ns=False), 'commonsfile': self.newname }) # If the image is uploaded under a different name, replace all # instances if self.imagePage.title(with_ns=False) != self.newname: imagebot = ImageRobot( generator=self.preloadingGen, oldImage=self.imagePage.title(with_ns=False), newImage=self.newname, summary=moveSummary, always=True, loose=True) imagebot.run() # If the user want to delete the page and # the user has sysops privilege, delete the page, otherwise # it will be marked for deletion. if self.delete_after_done: self.imagePage.delete(moveSummary, False) return
def transfer_image(self, sourceImagePage): """ Download image and its description, and upload it to another site. :return: the filename which was used to upload the image """ sourceSite = sourceImagePage.site pywikibot.output( '\n>>> Transfer {source} from {source.site} to {target}\n'.format( source=sourceImagePage, target=self.opt.target)) url = sourceImagePage.get_file_url() pywikibot.output('URL should be: ' + url) # localize the text that should be printed on image description page try: description = sourceImagePage.get() # try to translate license templates if (sourceSite.sitename, self.opt.target.sitename) in licenseTemplates: for old, new in licenseTemplates[( sourceSite.sitename, self.opt.target.sitename)].items(): new = '{{%s}}' % new old = re.compile('{{%s}}' % old) description = textlib.replaceExcept( description, old, new, ['comment', 'math', 'nowiki', 'pre']) description = i18n.twtranslate(self.opt.target, 'imagetransfer-file_page_message', { 'site': sourceSite, 'description': description }) description += '\n\n' description += sourceImagePage.getFileVersionHistoryTable() # add interwiki link if sourceSite.family == self.opt.target.family: description += '\n\n{}'.format(sourceImagePage) except NoPageError: pywikibot.output( 'Image does not exist or description page is empty.') except IsRedirectPageError: pywikibot.output('Image description page is redirect.') else: bot = UploadRobot(url=url, description=description, target_site=self.opt.target, url_encoding=sourceSite.encoding(), keep_filename=self.opt.keepname, verify_description=not self.opt.keepname, ignore_warning=self.opt.ignore_warning, force_if_shared=self.opt.force_if_shared) # try to upload if bot.skip_run(): return target_filename = bot.upload_file(url) if target_filename \ and self.opt.target.sitename == 'commons:commons': # upload to Commons was successful reason = i18n.twtranslate(sourceSite, 'imagetransfer-nowcommons_notice') # try to delete the original image if we have a sysop account if sourceSite.has_right('delete'): if sourceImagePage.delete(reason): return if sourceSite.lang in nowCommonsTemplate \ and sourceSite.family.name in config.usernames \ and sourceSite.lang in \ config.usernames[sourceSite.family.name]: # add the nowCommons template. pywikibot.output('Adding nowCommons template to ' + sourceImagePage.title()) sourceImagePage.put( sourceImagePage.get() + '\n\n' + nowCommonsTemplate[sourceSite.lang] % target_filename, summary=reason)
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: str """ url = '' description = [] summary = None keep_filename = False always = False use_filename = None filename_prefix = None verify_description = True aborts = set() ignorewarn = set() chunk_size = 0 recursive = False description_file = None # process all global bot args # returns a list of non-global args, i.e. args for upload.py local_args = pywikibot.handle_args(args) for option in local_args: arg, _, value = option.partition(':') if arg == '-always': keep_filename = True always = True verify_description = False elif arg == '-recursive': recursive = True elif arg == '-keep': keep_filename = True elif arg == '-filename': use_filename = value elif arg == '-prefix': filename_prefix = value elif arg == '-summary': summary = value elif arg == '-noverify': verify_description = False elif arg == '-abortonwarn': if value and aborts is not True: aborts.add(value) else: aborts = True elif arg == '-ignorewarn': if value and ignorewarn is not True: ignorewarn.add(value) else: ignorewarn = True elif arg == '-chunked': match = CHUNK_SIZE_REGEX.match(option) chunk_size = get_chunk_size(match) elif arg == '-descfile': description_file = value elif not url: url = option else: description.append(option) description = ' '.join(description) if description_file: if description: pywikibot.error('Both a description and a -descfile were ' 'provided. Please specify only one of those.') return False with codecs.open(description_file, encoding=pywikibot.config.textfile_encoding) as f: description = f.read().replace('\r\n', '\n') while not ('://' in url or os.path.exists(url)): if not url: error = 'No input filename given.' else: error = 'Invalid input filename given.' if not always: error += ' Try again.' if always: url = None break else: pywikibot.output(error) url = pywikibot.input('URL, file or directory where files are now:') if always and (aborts is not True and ignorewarn is not True or not description or url is None): additional = '' missing = [] if url is None: missing += ['filename'] additional = error + ' ' if description is None: missing += ['description'] if aborts is not True and ignorewarn is not True: additional += ('Either -ignorewarn or -abortonwarn must be ' 'defined for all codes. ') additional += 'Unable to run in -always mode' suggest_help(missing_parameters=missing, additional_text=additional) return False if os.path.isdir(url): file_list = [] for directory_info in os.walk(url): if not recursive: # Do not visit any subdirectories directory_info[1][:] = [] for dir_file in directory_info[2]: file_list.append(os.path.join(directory_info[0], dir_file)) url = file_list else: url = [url] print(url) bot = UploadRobot(url, description=description, useFilename=use_filename, keepFilename=keep_filename, verifyDescription=verify_description, aborts=aborts, ignoreWarning=ignorewarn, chunk_size=chunk_size, always=always, summary=summary, filename_prefix=filename_prefix) bot.run()
def processPhoto(flickr, photo_id=u'', flickrreview=False, reviewer=u'', override=u'', addCategory=u'', removeCategories=False, autonomous=False): """Process a single Flickr photo.""" if photo_id: pywikibot.output(str(photo_id)) (photoInfo, photoSizes) = getPhoto(flickr, photo_id) if isAllowedLicense(photoInfo) or override: # Get the url of the largest photo photoUrl = getPhotoUrl(photoSizes) # Should download the photo only once photo = downloadPhoto(photoUrl) # Don't upload duplicate images, should add override option duplicates = findDuplicateImages(photo) if duplicates: pywikibot.output(u'Found duplicate image at %s' % duplicates.pop()) else: filename = getFilename(photoInfo) flinfoDescription = getFlinfoDescription(photo_id) photoDescription = buildDescription(flinfoDescription, flickrreview, reviewer, override, addCategory, removeCategories) # pywikibot.output(photoDescription) if not isinstance(Tkdialog, ImportError) and not autonomous: try: (newPhotoDescription, newFilename, skip) = Tkdialog(photoDescription, photo, filename).show_dialog() except ImportError as e: pywikibot.warning(e) pywikibot.warning('Switching to autonomous mode.') autonomous = True elif not autonomous: pywikibot.warning('Switching to autonomous mode because GUI ' 'interface cannot be used') pywikibot.warning(Tkdialog) autonomous = True if autonomous: newPhotoDescription = photoDescription newFilename = filename skip = False # pywikibot.output(newPhotoDescription) # if (pywikibot.Page(title=u'File:'+ filename, site=pywikibot.Site()).exists()): # TODO: Check if the hash is the same and if not upload it under a different name # pywikibot.output(u'File:' + filename + u' already exists!') # else: # Do the actual upload # Would be nice to check before I upload if the file is already at Commons # Not that important for this program, but maybe for derived programs if not skip: bot = UploadRobot(photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) bot.upload_image(debug=False) return 1 else: pywikibot.output(u'Invalid license') return 0
def processPhoto(flickr, photo_id=u'', flickrreview=False, reviewer=u'', override=u'', addCategory=u'', removeCategories=False, autonomous=False): """Process a single Flickr photo.""" if photo_id: pywikibot.output(str(photo_id)) (photoInfo, photoSizes) = getPhoto(flickr, photo_id) if isAllowedLicense(photoInfo) or override: # Get the url of the largest photo photoUrl = getPhotoUrl(photoSizes) # Should download the photo only once photo = downloadPhoto(photoUrl) # Don't upload duplicate images, should add override option duplicates = findDuplicateImages(photo) if duplicates: pywikibot.output(u'Found duplicate image at %s' % duplicates.pop()) else: filename = getFilename(photoInfo) flinfoDescription = getFlinfoDescription(photo_id) photoDescription = buildDescription(flinfoDescription, flickrreview, reviewer, override, addCategory, removeCategories) # pywikibot.output(photoDescription) if not isinstance(Tkdialog, ImportError) and not autonomous: try: (newPhotoDescription, newFilename, skip) = Tkdialog( photoDescription, photo, filename).show_dialog() except ImportError as e: pywikibot.warning(e) pywikibot.warning('Switching to autonomous mode.') autonomous = True elif not autonomous: pywikibot.warning('Switching to autonomous mode because GUI ' 'interface cannot be used') pywikibot.warning(Tkdialog) autonomous = True if autonomous: newPhotoDescription = photoDescription newFilename = filename skip = False # pywikibot.output(newPhotoDescription) # if (pywikibot.Page(title=u'File:'+ filename, site=pywikibot.Site()).exists()): # TODO: Check if the hash is the same and if not upload it under a different name # pywikibot.output(u'File:' + filename + u' already exists!') # else: # Do the actual upload # Would be nice to check before I upload if the file is already at Commons # Not that important for this program, but maybe for derived programs if not skip: bot = UploadRobot(photoUrl, description=newPhotoDescription, useFilename=newFilename, keepFilename=True, verifyDescription=False) bot.upload_image(debug=False) return 1 else: pywikibot.output(u'Invalid license') return 0
def transferImage(self, sourceImagePage): """ Download image and its description, and upload it to another site. @return: the filename which was used to upload the image """ sourceSite = sourceImagePage.site url = sourceImagePage.fileUrl().encode('utf-8') pywikibot.output(u"URL should be: %s" % url) # localize the text that should be printed on the image description page try: description = sourceImagePage.get() # try to translate license templates if (sourceSite.sitename, self.targetSite.sitename) in licenseTemplates: for old, new in licenseTemplates[( sourceSite.sitename, self.targetSite.sitename)].items(): new = '{{%s}}' % new old = re.compile('{{%s}}' % old) description = textlib.replaceExcept( description, old, new, ['comment', 'math', 'nowiki', 'pre']) description = i18n.twtranslate( self.targetSite, 'imagetransfer-file_page_message', dict(site=sourceSite, description=description)) description += '\n\n' description += sourceImagePage.getFileVersionHistoryTable() # add interwiki link if sourceSite.family == self.targetSite.family: description += u'\r\n\r\n{0}'.format(sourceImagePage) except pywikibot.NoPage: description = '' pywikibot.output( 'Image does not exist or description page is empty.') except pywikibot.IsRedirectPage: description = '' pywikibot.output('Image description page is redirect.') else: bot = UploadRobot(url=url, description=description, targetSite=self.targetSite, urlEncoding=sourceSite.encoding(), keepFilename=self.keep_name, verifyDescription=not self.keep_name, ignoreWarning=self.ignore_warning) # try to upload targetFilename = bot.run() if targetFilename and self.targetSite.family.name == 'commons' and \ self.targetSite.code == 'commons': # upload to Commons was successful reason = i18n.twtranslate(sourceSite, 'imagetransfer-nowcommons_notice') # try to delete the original image if we have a sysop account if sourceSite.family.name in config.sysopnames and \ sourceSite.lang in config.sysopnames[sourceSite.family.name]: if sourceImagePage.delete(reason): return if sourceSite.lang in nowCommonsTemplate and \ sourceSite.family.name in config.usernames and \ sourceSite.lang in config.usernames[sourceSite.family.name]: # add the nowCommons template. pywikibot.output(u'Adding nowCommons template to %s' % sourceImagePage.title()) sourceImagePage.put( sourceImagePage.get() + '\n\n' + nowCommonsTemplate[sourceSite.lang] % targetFilename, summary=reason)
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ url = u'' description = [] summary = None keepFilename = False always = False useFilename = None verifyDescription = True aborts = set() ignorewarn = set() chunk_size_regex = re.compile( r'^-chunked(?::(\d+(?:\.\d+)?)[ \t]*(k|ki|m|mi)?b?)?$', re.I) recursive = False # process all global bot args # returns a list of non-global args, i.e. args for upload.py local_args = pywikibot.handle_args(args) for option in local_args: arg, _, value = option.partition(':') if arg == '-always': keepFilename = True always = True verifyDescription = False elif arg == '-recursive': recursive = True elif arg == '-keep': keepFilename = True elif arg == '-filename': useFilename = value elif arg == '-summary': summary = value elif arg == '-noverify': verifyDescription = False elif arg == '-abortonwarn': if value and aborts is not True: aborts.add(value) else: aborts = True elif arg == '-ignorewarn': if value and ignorewarn is not True: ignorewarn.add(value) else: ignorewarn = True elif arg == '-chunked': match = chunk_size_regex.match(option) chunk_size = get_chunk_size(match) elif arg and not value: if not url: url = arg else: description.append(arg) description = u' '.join(description) while not ("://" in url or os.path.exists(url)): if not url: error = 'No input filename given.' else: error = 'Invalid input filename given.' if not always: error += ' Try again.' if always: url = None break else: pywikibot.output(error) url = pywikibot.input(u'URL, file or directory where files are now:') if always and ((aborts is not True and ignorewarn is not True) or not description or url is None): additional = '' missing = [] if url is None: missing += ['filename'] additional = error + ' ' if description is None: missing += ['description'] if aborts is not True and ignorewarn is not True: additional += ('Either -ignorewarn or -abortonwarn must be ' 'defined for all codes. ') additional += 'Unable to run in -always mode' suggest_help(missing_parameters=missing, additional_text=additional) return False if os.path.isdir(url): file_list = [] for directory_info in os.walk(url): if not recursive: # Do not visit any subdirectories directory_info[1][:] = [] for dir_file in directory_info[2]: file_list.append(os.path.join(directory_info[0], dir_file)) url = file_list else: url = [url] bot = UploadRobot(url, description=description, useFilename=useFilename, keepFilename=keepFilename, verifyDescription=verifyDescription, aborts=aborts, ignoreWarning=ignorewarn, chunk_size=chunk_size, always=always, summary=summary) bot.run()
def transferImage(self, sourceImagePage): """ Download image and its description, and upload it to another site. @return: the filename which was used to upload the image """ sourceSite = sourceImagePage.site url = sourceImagePage.fileUrl().encode('utf-8') pywikibot.output(u"URL should be: %s" % url) # localize the text that should be printed on the image description page try: description = sourceImagePage.get() # try to translate license templates if (sourceSite.sitename, self.targetSite.sitename) in licenseTemplates: for old, new in licenseTemplates[ (sourceSite.sitename, self.targetSite.sitename)].items(): new = '{{%s}}' % new old = re.compile('{{%s}}' % old) description = textlib.replaceExcept(description, old, new, ['comment', 'math', 'nowiki', 'pre']) description = i18n.twtranslate(self.targetSite, 'imagetransfer-file_page_message', {'site': sourceSite, 'description': description}) description += '\n\n' description += sourceImagePage.getFileVersionHistoryTable() # add interwiki link if sourceSite.family == self.targetSite.family: description += u'\r\n\r\n{0}'.format(sourceImagePage) except pywikibot.NoPage: description = '' pywikibot.output( 'Image does not exist or description page is empty.') except pywikibot.IsRedirectPage: description = '' pywikibot.output('Image description page is redirect.') else: bot = UploadRobot(url=url, description=description, targetSite=self.targetSite, urlEncoding=sourceSite.encoding(), keepFilename=self.keep_name, verifyDescription=not self.keep_name, ignoreWarning=self.ignore_warning) # try to upload targetFilename = bot.run() if targetFilename and self.targetSite.family.name == 'commons' and \ self.targetSite.code == 'commons': # upload to Commons was successful reason = i18n.twtranslate(sourceSite, 'imagetransfer-nowcommons_notice') # try to delete the original image if we have a sysop account if sourceSite.family.name in config.sysopnames and \ sourceSite.lang in config.sysopnames[sourceSite.family.name]: if sourceImagePage.delete(reason): return if sourceSite.lang in nowCommonsTemplate and \ sourceSite.family.name in config.usernames and \ sourceSite.lang in config.usernames[sourceSite.family.name]: # add the nowCommons template. pywikibot.output(u'Adding nowCommons template to %s' % sourceImagePage.title()) sourceImagePage.put(sourceImagePage.get() + '\n\n' + nowCommonsTemplate[sourceSite.lang] % targetFilename, summary=reason)
def run(self): """Run the bot.""" tosend = {'language': self.imagePage.site.lang.encode('utf-8'), 'image': self.imagePage.title( withNamespace=False).encode('utf-8'), 'newname': self.newname.encode('utf-8'), 'project': self.imagePage.site.family.name.encode('utf-8'), 'username': '', 'commonsense': '1', 'remove_categories': '1', 'ignorewarnings': '1', 'doit': 'Uitvoeren' } tosend = urlencode(tosend) pywikibot.output(tosend) CH = pageTextPost('http://tools.wmflabs.org/commonshelper/index.php', tosend) pywikibot.output('Got CH desc.') tablock = CH.split('<textarea ')[1].split('>')[0] CH = CH.split('<textarea ' + tablock + '>')[1].split('</textarea>')[0] CH = CH.replace(u'×', u'×') CH = self.fixAuthor(CH) pywikibot.output(CH) # I want every picture to be tagged with the bottemplate so i can check # my contributions later. CH = ('\n\n{{BotMoveToCommons|' + self.imagePage.site.lang + '.' + self.imagePage.site.family.name + '|year={{subst:CURRENTYEAR}}|month={{subst:CURRENTMONTHNAME}}' '|day={{subst:CURRENTDAY}}}}' + CH) if self.category: CH = CH.replace('{{subst:Unc}} <!-- Remove this line once you have ' 'added categories -->', '') CH += u'[[Category:' + self.category + u']]' bot = UploadRobot(url=self.imagePage.fileUrl(), description=CH, useFilename=self.newname, keepFilename=True, verifyDescription=False, ignoreWarning=True, targetSite=pywikibot.Site('commons', 'commons')) bot.run() # Should check if the image actually was uploaded if pywikibot.Page(pywikibot.Site('commons', 'commons'), u'Image:' + self.newname).exists(): # Get a fresh copy, force to get the page so we dont run into edit # conflicts imtxt = self.imagePage.get(force=True) # Remove the move to commons templates if self.imagePage.site.lang in moveToCommonsTemplate: for moveTemplate in moveToCommonsTemplate[ self.imagePage.site.lang]: imtxt = re.sub(r'(?i)\{\{' + moveTemplate + r'[^\}]*\}\}', '', imtxt) # add {{NowCommons}} if self.imagePage.site.lang in nowCommonsTemplate: addTemplate = nowCommonsTemplate[ self.imagePage.site.lang] % self.newname else: addTemplate = nowCommonsTemplate['_default'] % self.newname commentText = i18n.twtranslate( self.imagePage.site, 'commons-file-now-available', {'localfile': self.imagePage.title(withNamespace=False), 'commonsfile': self.newname}) pywikibot.showDiff(self.imagePage.get(), imtxt + addTemplate) self.imagePage.put(imtxt + addTemplate, comment=commentText) self.gen = pagegenerators.FileLinksGenerator(self.imagePage) self.preloadingGen = pagegenerators.PreloadingGenerator(self.gen) # If the image is uploaded under a different name, replace all # instances if self.imagePage.title(withNamespace=False) != self.newname: moveSummary = i18n.twtranslate( self.imagePage.site, 'commons-file-moved', {'localfile': self.imagePage.title(withNamespace=False), 'commonsfile': self.newname}) imagebot = image.ImageRobot( generator=self.preloadingGen, oldImage=self.imagePage.title(withNamespace=False), newImage=self.newname, summary=moveSummary, always=True, loose=True) imagebot.run() return
def main(): site = pywikibot.Site('15mpedia', '15mpedia') flickrapilimit = 500 flickrapikey = getflickrapikey() #do not share key flickrseturl = "" categories = [] tags = [] #load parameters if len(sys.argv) > 1: for arg in sys.argv[1:]: if arg.startswith('--flickrset:'): # --flickrset:http://www.flickr.com/photos/15mmalagacc/sets/72157629844179358/ flickrseturl = arg[12:] elif arg.startswith('--categories:'): # --categories:"15M_en_Madrid;Ocupa_el_Congreso" categories = [re.sub('_', ' ', category) for category in arg[13:].split(';')] elif arg.startswith('--tags:'): # --tags:"15M;Acampada_Sol" tags = [re.sub('_', ' ', tag) for tag in arg[7:].split(';')] if not flickrseturl: print('Provide --flickrset: parameter. Example: --flickrset:https://www.flickr.com/photos/15mmalagacc/sets/72157629844179358/') sys.exit() """if not categories: print('Provide --categories: parameter. Example: --categories:"15M_en_Madrid;Ocupa_el_Congreso"') sys.exit()""" flickrseturls = [] if '://' in flickrseturl: if '/people/' in flickrseturl: raw = getURL(url=flickrseturl) flickruser = flickrseturl.split('/people/')[1].split('/')[0].strip('/') flickruserid = re.findall(r'"nsid":"([^"]+)"', raw)[0] apiquery = 'https://api.flickr.com/services/rest/?method=flickr.photosets.getList&api_key=%s&user_id=%s&format=json&nojsoncallback=1' % (flickrapikey, flickruserid) jsonset3 = json.loads(getURL(url=apiquery)) if not "photosets" in jsonset3: print("ERROR: API key caducada o invalida?") sys.exit() flickrseturls = ["https://www.flickr.com/photos/%s/albums/%s" % (flickruser, x["id"]) for x in jsonset3["photosets"]["photoset"]] print('\n'.join(flickrseturls)) sys.exit() else: flickrseturls = [flickrseturl] else: f = open(flickrseturl, 'r') flickrseturls = f.read().strip().splitlines() f.close() print('Loaded', len(flickrseturls), 'photosets') for flickrseturl in flickrseturls: flickrseturl = flickrseturl.replace('/sets/', '/albums/') flickruser = flickrseturl.split('/photos/')[1].split('/albums/')[0].strip() flickrsetid = flickrseturl.split('/albums/')[1].split('/')[0].strip('/').strip() raw = getURL(url=flickrseturl) m = re.findall(r'"albumId":"%s","nsid":"([^"]+?)"' % (flickrsetid), raw) flickruserid = '' if m: flickruserid = m[0] else: print("No se encontro flickruserid") sys.exit() m = re.findall(r',"pathAlias":"([^"]+?)"', raw) #load set metadata apiquery = 'https://api.flickr.com/services/rest/?method=flickr.photosets.getPhotos&api_key=%s&photoset_id=%s&user_id=%s&per_page=%s&format=json&nojsoncallback=1' % (flickrapikey, flickrsetid, flickruserid, flickrapilimit) jsonset = json.loads(getURL(url=apiquery)) if not "photoset" in jsonset: print("ERROR: API key caducada o invalida?") sys.exit() #print(jsonset) flickrsetname = jsonset["photoset"]["title"] #flickruser = jsonset["photoset"]["ownername"] #hay usuarios con espacios, mejor extraer el usuario de la url del set photoids = [photo["id"] for photo in jsonset["photoset"]["photo"]] pages = int(jsonset["photoset"]["pages"]) if pages > 1: for page in range(2, pages+1): apiquery2 = apiquery + '&page=' + str(page) jsonset2 = json.loads(getURL(url=apiquery2)) photoids += [photo["id"] for photo in jsonset2["photoset"]["photo"]] print('There are', len(photoids), 'images in the set', flickrsetid, 'by', flickruser) #load images metadata for photoid in photoids: apiquery = 'https://api.flickr.com/services/rest/?method=flickr.photos.getInfo&api_key=%s&photo_id=%s&format=json&nojsoncallback=1' % (flickrapikey, photoid) jsonphoto = json.loads(getURL(url=apiquery)) #check license, if not free, do not donwload later photolicense = jsonphoto["photo"]["license"] if not photolicense in ["1", "2", "3", "4", "5", "6", "9", "10"]: print('Skiping', photoid, 'which is not Creative Commons or Public Domain') continue photometadata = { 'title': unquote("title" in jsonphoto["photo"] and jsonphoto["photo"]["title"]["_content"].strip() or ''), 'description': unquote("description" in jsonphoto["photo"] and jsonphoto["photo"]["description"]["_content"].strip() or ''), 'date-taken': "taken" in jsonphoto["photo"]["dates"] and jsonphoto["photo"]["dates"]["taken"] or '', 'license': photolicense, 'coordinates': "location" in jsonphoto["photo"] and [jsonphoto["photo"]["location"]["latitude"], jsonphoto["photo"]["location"]["longitude"]] or '', 'localfilename': '%s - %s - %s.jpg' % (flickruser, flickrsetid, photoid), 'photourl': "https://www.flickr.com/photos/%s/%s/" % (flickruser, photoid), 'tags': [tag["raw"] for tag in jsonphoto["photo"]["tags"]["tag"]] + tags, } photometadata['description'] = re.sub(r' *\n+ *', r'\n\n', photometadata['description']) if 'has uploaded' in photometadata['description']: photometadata['description'] = '' photofullres = 'https://farm%s.staticflickr.com/%s/%s_%s_o_d.jpg' % (jsonphoto["photo"]["farm"], jsonphoto["photo"]["server"], jsonphoto["photo"]["id"], jsonphoto["photo"]["originalsecret"]) print(photoid) print(photometadata) print(photofullres) cats = '' if categories: cats = '\n\n%s' % ('\n'.join(['[[Categoría:%s]]' % (category) for category in categories])) output = generateInfobox(photoid, photometadata, cats, flickrseturl, flickrsetname, flickruser) #https://www.mediawiki.org/wiki/Manual:Pywikibot/upload.py aborts = set() ignorewarn = set(['duplicate']) # los duplicados los controlamos con page.exists() mas abajo summary = "BOT - Subiendo imagen %s" % (photometadata['photourl']) page = pywikibot.Page(site, "File:%s" % (photometadata['localfilename'])) if page.exists(): print("Ya existe") continue #print(output) bot = UploadRobot(photofullres, description=output, useFilename=photometadata['localfilename'], keepFilename=True, verifyDescription=False, aborts=aborts, ignoreWarning=ignorewarn, summary=summary, targetSite=site) bot.run()
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ url = u'' description = [] summary = None keepFilename = False always = False useFilename = None verifyDescription = True aborts = set() ignorewarn = set() chunk_size = 0 chunk_size_regex = r'^-chunked(?::(\d+(?:\.\d+)?)[ \t]*(k|ki|m|mi)?b?)?$' chunk_size_regex = re.compile(chunk_size_regex, re.I) recursive = False # process all global bot args # returns a list of non-global args, i.e. args for upload.py for arg in pywikibot.handle_args(args): if arg: if arg == '-always': keepFilename = True always = True verifyDescription = False elif arg == '-recursive': recursive = True elif arg.startswith('-keep'): keepFilename = True elif arg.startswith('-filename:'): useFilename = arg[10:] elif arg.startswith('-summary'): summary = arg[9:] elif arg.startswith('-noverify'): verifyDescription = False elif arg.startswith('-abortonwarn'): if len(arg) > len('-abortonwarn:') and aborts is not True: aborts.add(arg[len('-abortonwarn:'):]) else: aborts = True elif arg.startswith('-ignorewarn'): if len(arg) > len('-ignorewarn:') and ignorewarn is not True: ignorewarn.add(arg[len('-ignorewarn:'):]) else: ignorewarn = True elif arg.startswith('-chunked'): match = chunk_size_regex.match(arg) if match: if match.group(1): # number was in there base = float(match.group(1)) if match.group(2): # suffix too suffix = match.group(2).lower() if suffix == "k": suffix = 1000 elif suffix == "m": suffix = 1000000 elif suffix == "ki": suffix = 1 << 10 elif suffix == "mi": suffix = 1 << 20 else: pass # huh? else: suffix = 1 chunk_size = math.trunc(base * suffix) else: chunk_size = 1 << 20 # default to 1 MiB else: pywikibot.error('Chunk size parameter is not valid.') elif url == u'': url = arg else: description.append(arg) description = u' '.join(description) # curly barckets need to double in formatted string description = """=={{{{int:filedesc}}}}== {{{{Information |description={{{{en|1=Native Israeli pronunciation of this Hebrew word}}}} |date={0} |source={{{{own}}}} |author=[[User:{1}|{1}]] |permission= |other versions= }}}} =={{{{int:license-header}}}}== {{{{self|cc-zero}}}} [[Category:Hebrew pronunciation]]""".format(date.today(),config.usernames['commons']['commons']) while not ("://" in url or os.path.exists(url)): if not url: error = 'No input filename given.' else: error = 'Invalid input filename given.' if not always: error += ' Try again.' if always: url = None break else: pywikibot.output(error) url = pywikibot.input(u'URL, file or directory where files are now:') if always and ((aborts is not True and ignorewarn is not True) or not description or url is None): additional = '' missing = [] if url is None: missing += ['filename'] additional = error + ' ' if description is None: missing += ['description'] if aborts is not True and ignorewarn is not True: additional += ('Either -ignorewarn or -abortonwarn must be ' 'defined for all codes. ') additional += 'Unable to run in -always mode' suggest_help(missing_parameters=missing, additional_text=additional) return False if os.path.isdir(url): file_list = [] for directory_info in os.walk(url): if not recursive: # Do not visit any subdirectories directory_info[1][:] = [] for dir_file in directory_info[2]: file_list.append(os.path.join(directory_info[0], dir_file)) url = file_list else: url = [url] bot = UploadRobot(url, description=description, useFilename=useFilename, keepFilename=keepFilename, verifyDescription=verifyDescription, aborts=aborts, ignoreWarning=ignorewarn, chunk_size=chunk_size, always=always, summary="bot upload", targetSite=pywikibot.Site('commons', 'commons')) bot.run()
def main(*args): """ Process command line arguments and invoke bot. If args is an empty list, sys.argv is used. @param args: command line arguments @type args: list of unicode """ url = u'' description = [] summary = None keepFilename = False always = False useFilename = None verifyDescription = True aborts = set() ignorewarn = set() chunk_size = 0 chunk_size_regex = r'^-chunked(?::(\d+(?:\.\d+)?)[ \t]*(k|ki|m|mi)?b?)?$' chunk_size_regex = re.compile(chunk_size_regex, re.I) recursive = False # process all global bot args # returns a list of non-global args, i.e. args for upload.py for arg in pywikibot.handle_args(args): if arg: if arg == '-always': keepFilename = True always = True verifyDescription = False elif arg == '-recursive': recursive = True elif arg.startswith('-keep'): keepFilename = True elif arg.startswith('-filename:'): useFilename = arg[10:] elif arg.startswith('-summary'): summary = arg[9:] elif arg.startswith('-noverify'): verifyDescription = False elif arg.startswith('-abortonwarn'): if len(arg) > len('-abortonwarn:') and aborts is not True: aborts.add(arg[len('-abortonwarn:'):]) else: aborts = True elif arg.startswith('-ignorewarn'): if len(arg) > len('-ignorewarn:') and ignorewarn is not True: ignorewarn.add(arg[len('-ignorewarn:'):]) else: ignorewarn = True elif arg.startswith('-chunked'): match = chunk_size_regex.match(arg) if match: if match.group(1): # number was in there base = float(match.group(1)) if match.group(2): # suffix too suffix = match.group(2).lower() if suffix == "k": suffix = 1000 elif suffix == "m": suffix = 1000000 elif suffix == "ki": suffix = 1 << 10 elif suffix == "mi": suffix = 1 << 20 else: pass # huh? else: suffix = 1 chunk_size = math.trunc(base * suffix) else: chunk_size = 1 << 20 # default to 1 MiB else: pywikibot.error('Chunk size parameter is not valid.') elif url == u'': url = arg else: description.append(arg) description = u' '.join(description) while not ("://" in url or os.path.exists(url)): if not url: error = 'No input filename given.' else: error = 'Invalid input filename given.' if not always: error += ' Try again.' if always: url = None break else: pywikibot.output(error) url = pywikibot.input(u'URL, file or directory where files are now:') if always and ((aborts is not True and ignorewarn is not True) or not description or url is None): additional = '' missing = [] if url is None: missing += ['filename'] additional = error + ' ' if description is None: missing += ['description'] if aborts is not True and ignorewarn is not True: additional += ('Either -ignorewarn or -abortonwarn must be ' 'defined for all codes. ') additional += 'Unable to run in -always mode' suggest_help(missing_parameters=missing, additional_text=additional) return False if os.path.isdir(url): file_list = [] for directory_info in os.walk(url): if not recursive: # Do not visit any subdirectories directory_info[1][:] = [] for dir_file in directory_info[2]: file_list.append(os.path.join(directory_info[0], dir_file)) url = file_list else: url = [url] bot = UploadRobot(url, description=description, useFilename=useFilename, keepFilename=keepFilename, verifyDescription=verifyDescription, aborts=aborts, ignoreWarning=ignorewarn, chunk_size=chunk_size, always=always, summary=summary) bot.run()