#!/usr/bin/python

# -*- coding: utf-8 -*-

'''

# Check and update Xeno-Canto audio files

# Date: July 2013

# Author: Fae, http://j.mp/faewm

# CC-BY-SA

'''

import wikipedia, upload, sys, config, urllib2, urllib, re, string, time, catlib, pagegenerators, os.path, hashlib, pprint

import json,subprocess

from BeautifulSoup import BeautifulSoup

from sys import argv, stdout

import collections

from time import sleep

from os import remove

from colorama import Fore, Back, Style, init

init()

site = wikipedia.getSite('commons', 'commons')

subs=[['%20',' '],['%28','('],['%29',')'],['%2C',','],['%3A',':']]

def pquote(s):

return s

def gettag(tag,name):

return ''

def p(tag, val): # Print nicely

if len(val)>1:

print Fore.CYAN+tag+" : "+Fore.YELLOW+val+Fore.WHITE

return

def pp(tag, val): # Print nicely

return

def trim(s): # Trim leading and trailing spaces

return re.sub('^[\s\r\t\f\n]*|[\s\r\t\f\n]*$','',s)

def up(filename, pagetitle, desc):

bot.upload_image(debug=True)

def urltry(u):

return x

def htmltry(x,u):

return

def xmlfind(item):

return ''

def gettag(h):

return ", ".join(t)

def getatt(t,h):

return re.sub('^\s*|<[^>]*>|\s*$','',r)

def getcat(h):

return ", ".join(r)

def dupcheck(ff): # Using the SHA1 checksum, find if the file is already uploaded to Commons

return False,''

# Check if a file title is already used on commons

def nameused(name):

return False

def trimtags(s):

return trim(re.sub('<[^>]*?>','',s))

def titlecase(s):

return ' '.join(words)

def oddchars(s):

return s

def asciichars(s):

return s

# *** Grab description ***

def flatten(l):

yield el

def getdesc(html):

return '{{information\n| description = {{en|1=<br/>\n:''Image title: '+image['title']+'\n:Image from Public domain images website, '+site+"}}\n| source = "+image['src']+"\n| author = "+author+'\n| date = Not given\n*Transferred by [[User:{{subst:User:Fae/Fae}}|]] on {{subst:today}}\n| permission = This file is in public domain, not copyrighted, no rights reserved, free for any use. You can use this picture for any use including commercial purposes without the prior written permission and without fee or obligation.\n}}\n=={{int:license}}==\n{{PD-author|1='+author+'}}\n[[Category:Images uploaded by {{subst:User:Fae/Fae}}]]\n[[Category:Public-domain-image.com]]'+allcat, re.sub("\s{2,}"," ",image['title'].encode('ascii','ignore'))+'.jpg', image['src']

def uptry(source,filename,desc):

return

plist=[]

def numberSearch(v):

return vcount

def catexists(cat): # Does this Commons category exist?

return True

def createcat(cat,txt,action):

return

def sleepcounter(period):

stdout.write("\r")

def ns(s):

return re.sub(" ","_",s)

# *********

# Main loop

# *********

baseurl="http://www.xeno-canto.org/api/recordings.php?query=lic:by-sa&page="

page=1

uri=baseurl+str(page)

print "Getting",uri

url=urltry(uri)

html=json.load(url)

print str(html)[0:100]

# error was a key, not sure what happened to it

'''

if html['error']==0:

print Fore.GREEN,"Loaded without errors",Fore.WHITE

else:

print Fore.RED,"Error",html['errorMessage'],Fore.WHITE

'''

results=html

recordings=results['recordings'] #[i for i in html['results']['recordings']]

numRecordings=results['numRecordings']

numPages=results['numPages']

print Fore.YELLOW,"numRecordings",numRecordings,"\n numPages",numPages,Fore.WHITE

for p in range(int(float(numPages))):

if p==0: continue

uri=baseurl+str(p+1)

print "Getting",uri

url=urltry(uri)

html2=json.load(url)

#print hArr

'''if html2['error']==0:

print Fore.GREEN,"Loaded without errors",Fore.WHITE

else:

print Fore.RED,"Error",html2['errorMessage'],Fore.WHITE'''

#for i in html2['results']['recordings']:

recordings=html2['recordings']#.append(i)

print Fore.CYAN+"Number of recordings identified:",len(recordings),"of",numRecordings

count=0

results=[]

for r in recordings:

count+=1

#pprint.PrettyPrinter(indent=2).pprint(r)

# Quick check of uniqueness of id on Commons, for virin read filename

plist=[]

ref="XC"+r['id']

title=r['en']

numbermatch='"xeno-canto"+"'+r['gen']+'"+"'+r['sp']+'"+"'+ref+'"'

vc=numberSearch(numbermatch)

print Fore.YELLOW+str(count), Fore.CYAN+"http://commons.wikimedia.org/w/index.php?search="+urllib.quote(numbermatch), Fore.WHITE

if vc==-1:

print Fore.YELLOW+"Error generating search matches for '"+title+" "+ref+"', probably timeout error, skipping file without completing this test",Fore.WHITE

continue

if vc>0: # May be a need to ignore known false positives

#print Fore.RED+"-"*75

print Fore.RED+"Image appears to be in use already for",asciichars(oddchars(title))+" "+ref+" ("+str(vc)+" matches)."+Fore.WHITE

#print Fore.RED+"-"*75,Fore.WHITE

continue

else:

results.append(r)

workingdir="F:/XC/"

count=0

uploadcount=0

for r in results:

count+=1

ref=re.sub("\D*","",r['id'])

species=r['sp']

genus=r['gen']

common=r['en']

title=genus+" "+species+" - "+common+" XC"+ref

filename=asciichars(oddchars(title))+".ogg" # upload filename

# Quick check of filename in use - these should be unique

if nameused(filename):

print Fore.RED+'Filename found',Fore.YELLOW+"http://commons.wikimedia.org/wiki/File:"+re.sub("%20","_",urllib.quote(filename)),Fore.WHITE

continue

localfile=workingdir+ref+".mp3" # source mp3 file

localenc=workingdir+ref

print Fore.GREEN+filename+Fore.WHITE

source=r['file']

artist=r['rec']

gallery=r['url']

url=urltry(gallery)

html=htmltry(url,gallery)

soup=BeautifulSoup(html)

rd=str(soup.find('section',{'id':'recording-data'}).find('tbody'))

date=rd.split(">Date<")[1].split('<td>')[1].split('<')[0]

dtime=""

if re.search(">Time<",rd):

dtime=rd.split(">Time<")[1].split('<td>')[1].split('<')[0]

if len(dtime)>2:

date+=" "+dtime

elevation=''

if re.search('>Elevation<',rd):

elevation=rd.split(">Elevation<")[1].split('<td>')[1].split('<')[0]

background=''

if re.search('>Background<',rd):

background=rd.split(">Background")[1].split('<td')[1].split('>')[1].split('<')[0]

if background=="none":

background=''

remarks=''

if re.search('<h2>Remarks from',html):

remarks=html.split('<h2>Remarks')[1].split('h2>')[1]

if re.search('<.section',remarks):

remarks=remarks.split('</section')[0]

remarks=re.sub("<.?p>","\n",remarks)

remarks=re.sub("<.?span[^>]*?>","\n",remarks)

remarks=re.sub("\n*\s*<$","",remarks)

remarks=re.sub("\n+","\n:",re.sub("\s*\n\s*","\n",remarks))

remarks=re.sub("<a [^>]*>","",re.sub("<\/a>","",remarks))

d="=={{int:filedesc}}==\n{{information"

d+="\n|description={{en|"

if remarks!='':

d+=remarks

if len(r['en'])>2:

d+="\n:'''Common name:''' "+r['en']

if len(r['type'])>2:

d+="\n:'''Type:''' "+r['type']

if len(r['gen'])>2:

d+="\n:'''Genus:''' "+r['gen']

if len(r['sp'])>2:

d+="\n:'''Species:''' "+r['sp']

if len(r['loc'])>2:

d+="\n:'''Location:''' "+r['loc']

if len(r['cnt'])>2:

d+="\n:'''Country:''' "+r['cnt']

if elevation!='':

d+="\n:'''Elevation:''' "+elevation

if background!='':

d+="\n:Background:''' "+background

d+="}}"

d+="\n|date="+date

d+="\n|author="+artist

d+="\n|source=\n*'''Metadata:''' "+gallery+"\n*'''Audio file:''' "+source

d+="\n}}"

if len(r['lat'])>2:

d+="{{object location dec|"+r['lat']+"|"+r['lng']+"}}"

d+="{{User:{{subst:User:Fae/Fae}}/Projects/Xeno-canto/credit}}"

d+="\n\n=={{int:license-header}}==\n{{cc-by-sa-3.0}}"

d+="\n[[Category:Xeno-canto]]\n[[Category:Sound files uploaded by {{subst:User:Fae/Fae}}]]"

cat=r['gen']+" "+r['sp']

if catexists(cat):

d+="\n[[Category:"+cat+"]]"

d=re.sub("ocolono",":",oddchars(re.sub(":","ocolono",d)))

print Fore.CYAN+d,Fore.WHITE

urllib.urlretrieve(source, localfile)

print Fore.YELLOW+str(count)+"/"+str(len(results)),Fore.GREEN,"Encoding",localfile

print "ffmpeg -i",localfile,localenc+".wav",Fore.WHITE

subprocess.call(["ffmpeg","-i",localfile,localenc+".wav"])

subprocess.call(["oggenc2",localenc+".wav",

"-q 10", #quality

"-a "+ns(artist), #artist

"-l "+ns(gallery), #album

"-t "+ns(title), #title

#"-d "+re.sub("\D*","",date.split(" ")[0]), #date

"-N "+ref, #track num

"-G "+ns(r['type']), #genre

"-c country="+ns(r['cnt']),

"-c location="+ns(r['loc']),

"-c genus="+ns(r['gen']),

"-c species="+ns(r['sp']),

"-c latitude="+ns(r['lat']),

"-c longitude="+ns(r['lng']),

"-c license="+ns(r['lic']),

"-c URL="+ns(r['url']),

"-c date="+date.split(" ")[0],

"-c time="+dtime,

"-o",localenc+".ogg"])

remove(localenc+".wav")

remove(localenc+".mp3")

# Check if image is a duplicate on Commons

media=localenc+".ogg"

try:

duplicate,duptitle = dupcheck(media)

except:

print Fore.RED+"Problem when running the duplicate check for",media,Fore.WHITE

time.sleep(10)

try:

duplicate,duptitle = dupcheck(media)

except:

print Fore.RED+"Failed on second try, giving up and skipping"

remove(media)

continue

if duplicate:

print Fore.RED+'File is already on Commons as',duptitle,Fore.WHITE

time.sleep(2)

continue

# Upload

uptry(media,filename,d) # Intermittently getting [Errno 54] Connection reset by peer

remove(media)

uploadcount+=1

print Fore.GREEN+"Total uploads =",uploadcount,Fore.WHITE

sleepcounter(10)