def uploadFile(self):
'''Store pdf in package, gets sides from pdf, if self.sides
isn't empty
'''
filePath = self.path
log.debug(u"uploadFile " + unicode(filePath))
if not self.parentNode or not self.parentNode.package:
log.error('something is wrong with the file')
## replace all non-digits and non-usefull stuff with ''
self.pages = sub('[^\d,-]', '', self.pages)
if self.pages != "":
input = PdfFileReader(file(filePath, "rb"))
lastPage = input.getNumPages() - 1 # last page
toimport = PdfIdevice.__parseImportPages(self.pages, lastPage)
log.debug("Parsed pages: " + str(toimport))
output = PdfFileWriter()
for page in toimport:
output.addPage(input.getPage(page))
log.debug("Found pages to import %s" % toimport)
tmp = os.tmpnam() + ".pdf"
log.debug('Tempfile is %s' % tmp)
outputStream = file(tmp, "wb")
output.write(outputStream)
outputStream.close()
resourceFile = Path(tmp)
self.file = Resource(self, resourceFile)
log.debug("Uploaded %s, pages: %s" % (tmp, toimport))
os.remove(tmp)
filePath = tmp
resourceFile = Path(filePath)
if resourceFile.isfile():
self.file = Resource(self, resourceFile)
log.debug(u"uploaded " + self.path)
def testCreateAndDelete(self):
"""
Test we have a resource directory and resource files can be stored in
"""
oliver = Resource(self.package, Path("oliver.jpg"))
self.assert_((self.package.resourceDir/"oliver.jpg").exists())
oliver.delete()
self.assert_(not (self.package.resourceDir/"oliver.jpg").exists())
def testCreateAndDelete(self):
"""
Test we have a resource directory and resource files can be stored in
"""
myIdevice = Idevice("My Idevice", "UoA", "Testing", "Help tip", "icon", self.package.root)
oliver = Resource(myIdevice, Path("oliver.jpg"))
self.assert_((self.package.resourceDir/"oliver.jpg").exists())
oliver.delete()
self.assert_(not (self.package.resourceDir/"oliver.jpg").exists())
def testCreateAndDelete(self):
"""
Test we have a resource directory and resource files can be stored in
"""
myIdevice = Idevice("My Idevice", "UoA", "Testing", "Help tip", "icon",
self.package.root)
oliver = Resource(myIdevice, Path("oliver.jpg"))
self.assert_((self.package.resourceDir / "oliver.jpg").exists())
oliver.delete()
self.assert_(not (self.package.resourceDir / "oliver.jpg").exists())
class FileField(Field):
"""
Field for storing an individual file
"""
persistenceVersion = 5
"""
alwaysNameTo - make sure that this file always a certain final name
"""
def __init__(self,
idevice,
alwaysNameTo=None,
desc="File Field",
help="File Field Help"):
Field.__init__(self, desc, help)
self.idevice = idevice
self.fileResource = None
self.fileInstruc = "Upload a file"
self.alwaysNameTo = alwaysNameTo
self.fileDescription = TextField("Description")
self.fileDescription.idevice = self
def uploadFile(self, filePath):
if self.fileResource is not None:
self.fileResource.delete()
finalName = str(filePath)
if self.alwaysNameTo is not None:
from os.path import dirname
from shutil import copyfile
dirName = dirname(filePath)
finalName = dirName + "/" + self.alwaysNameTo
copyfile(filePath, finalName)
if self.fileDescription.content == "":
self.fileDescription.content = os.path.basename(filePath)
resourceFile = Path(finalName)
if resourceFile.isfile():
self.idevice.message = ""
self.fileResource = Resource(self.idevice, resourceFile)
def deleteFile(self):
if self.fileResource is not None:
self.fileResource.delete()
self.fileResource = None
def upgradeToVersion4(self):
self.fileDescription = TextField("Description")
def upgradeToVersion5(self):
self.fileDescription.idevice = self
class FileField(Field):
"""
Field for storing an individual file
"""
persistenceVersion = 5
"""
alwaysNameTo - make sure that this file always a certain final name
"""
def __init__(self, idevice, alwaysNameTo=None, desc="File Field", help="File Field Help"):
Field.__init__(self, desc, help)
self.idevice = idevice
self.fileResource = None
self.fileInstruc = "Upload a file"
self.alwaysNameTo = alwaysNameTo
self.fileDescription = TextField("Description")
self.fileDescription.idevice = self
def uploadFile(self, filePath):
if self.fileResource is not None:
self.fileResource.delete()
finalName = str(filePath)
if self.alwaysNameTo is not None:
from os.path import dirname
from shutil import copyfile
dirName = dirname(filePath)
finalName = dirName + "/" + self.alwaysNameTo
copyfile(filePath, finalName)
if self.fileDescription.content == "":
self.fileDescription.content = os.path.basename(filePath)
resourceFile = Path(finalName)
if resourceFile.isfile():
self.idevice.message = ""
self.fileResource = Resource(self.idevice, resourceFile)
def deleteFile(self):
if self.fileResource is not None:
self.fileResource.delete()
self.fileResource = None
def upgradeToVersion4(self):
self.fileDescription = TextField("Description")
def upgradeToVersion5(self):
self.fileDescription.idevice = self
def check_nodeResourcesChecksums(self):
for node in self.nodes:
for idevice in node.idevices:
for resource in idevice.userResources:
try:
nresource = Resource(self.tmppackage, resource.path)
except:
msg = '%s not exists in elp' % resource.storageName
self.appendInconsistency(msg,
'nodeResourceNonExistant',
idevice, resource)
continue
if nresource.checksum in self.package.resources.keys():
if nresource.checksum == resource.checksum:
continue
else:
msg = '%s checksum not consistent' % resource.storageName
self.appendInconsistency(
msg, 'nodeResourceNotInResources',
self.package, idevice, nresource.checksum,
resource)
else:
msg = '%s checksum not in package resources' % resource.storageName
self.appendInconsistency(msg,
'nodeResourceNotInResources',
self.package, idevice,
nresource.checksum, resource)
def check_packageResourcesNonReferenced(self):
for path in self.package.resourceDir.files():
resource = Resource(self.tmppackage, path)
if resource.checksum not in self.package.resources:
msg = '%s not in package resources' % resource.storageName
self.appendInconsistency(msg, 'packageResourceNonReferenced',
self.package, path)
def check_packageResourcesChecksums(self):
itype = 'packageResourceChecksum'
for checksum, resources in self.package.resources.items():
if resources:
for resource in resources:
try:
nresource = Resource(self.tmppackage, resource.path)
except:
msg = '%s not exists in elp' % resource.storageName
self.appendInconsistency(msg, itype, self.package,
checksum, resource)
continue
if nresource.checksum == checksum:
if nresource.checksum == resource.checksum:
continue
else:
msg = '%s checksum not consistent' % resource.storageName
self.appendInconsistency(msg, itype, self.package,
checksum, resource)
else:
msg = '%s checksum not consistent' % resource.storageName
self.appendInconsistency(msg, itype, self.package,
checksum, resource)
else:
self.package.resources.pop(checksum)
def check(self):
if self.clear:
log.info('Clearing resource references')
self.clear_resourceReferences()
log.info('Adding resource references')
for path in self.package.resourceDir.files():
if path in self.idevices:
for idevice in self.idevices[path]:
try:
resource = Resource(idevice, path)
except:
msg = "%s referenced in idevice %s of node %s not exists" % (
path, idevice.idevice.klass,
idevice.parentNode.title)
log.error(
'New inconsistency of type packageResourceNonExistant: %s'
% (msg))
continue
if isinstance(idevice, FieldWithResources):
galleryimage = GalleryImage(idevice,
'',
None,
mkThumbnail=False)
galleryimage._imageResource = resource
if isinstance(
idevice, Idevice
) and idevice.klass == 'ImageMagnifierIdevice':
idevice.imageMagnifier.imageResource = resource
if isinstance(
idevice,
Idevice) and idevice.klass == 'GalleryIdevice':
for image in idevice.images:
if image._imageResource.storageName == resource.storageName:
image._imageResource = resource
break
elif image._thumbnailResource.storageName == resource.storageName:
image._thumbnailResource = resource
break
elif self.package._backgroundImg and path == self.package._backgroundImg.path:
self.package._backgroundImg = Resource(self.package, path)
for check in dir(self):
if check.startswith('check_'):
fn = getattr(self, check)
log.info('Checking %s' % check[6:])
fn()
return self.inconsistencies
def upgradeToVersion3(self):
"""
Upgrades to v0.12
"""
self._upgradeIdeviceToVersion2()
if self.filename and self.parentNode:
Resource(self, Path(self.filename))
del self.filename
def upgradeToVersion6(self):
"""
Upgrades to v0.12
"""
self._upgradeIdeviceToVersion2()
self.systemResources += ["fdl.html"]
if self.images and self.parentNode:
for image in self.images:
imageResource = Resource(self, Path(image))
def __init__(self, baseurl, node, client=None):
self.baseurl = baseurl.decode(sys.getfilesystemencoding())
self.node = node
self.client = client
self.numdirs = 0
resources = {}
resources['mimes'] = {}
resources['urls'] = {}
url = Url(self.baseurl, self.baseurl)
url.createNode(node, _('Contents of directory'))
resources['urls'][url.relpath] = url
try:
for root, dirs, files in self._safewalk(self.baseurl):
if self.cancel:
return
self.numdirs += 1
except UnicodeDecodeError:
raise
i = 1
for root, dirs, files in self._safewalk(self.baseurl):
html = u""
idevice = None
if self.client:
self.client.call('eXe.app.getController("Toolbar").updateImportProgressWindow',_(u'Analizing directory %d of %d: %s') % (i, self.numdirs,root.encode(sys.getfilesystemencoding())))
for dir in dirs:
if self.cancel:
return
path = root + os.path.sep + dir
url = Url(path, self.baseurl)
url.createNode(resources['urls'][url.parentpath].node)
resources['urls'][url.relpath] = url
for file in files:
if self.cancel:
return
path = root + os.path.sep + file
url = Url(path, self.baseurl)
parent = resources['urls'][url.parentpath]
if not idevice:
idevice = parent.createIdevice()
try:
p = Path(path)
p.setSalt(str(url))
r = Resource(idevice,p)
except:
continue
url.href = 'resources/%s' % (quote(r.storageName))
html += u"<p><a href=%s>%s</p>\n" % (url.href,url.basename)
resources['urls'][url.relpath] = url
if url.mime in resources['mimes'].keys():
resources['mimes'][url.mime].append(url)
else:
resources['mimes'][url.mime] = [ url ]
if idevice:
idevice.setContent(html)
i += 1
self.resources = resources
def uploadNeededScripts(self):
from exe import globals
import os, sys
scriptFileNames = ['jquery-ui-1.10.3.custom.min.js', 'memmatch-0.2.js']
for scriptName in scriptFileNames:
from exe import globals
scriptSrcFilename = globals.application.config.webDir / "templates" / scriptName
gameScriptFile = Path(scriptSrcFilename)
if gameScriptFile.isfile():
Resource(self, gameScriptFile)
def uploadFile(self, filePath):
if self.fileResource is not None:
self.fileResource.delete()
finalName = str(filePath)
if self.alwaysNameTo is not None:
from os.path import dirname
from shutil import copyfile
dirName = dirname(filePath)
finalName = dirName + "/" + self.alwaysNameTo
copyfile(filePath, finalName)
if self.fileDescription.content == "":
self.fileDescription.content = os.path.basename(filePath)
resourceFile = Path(finalName)
if resourceFile.isfile():
self.idevice.message = ""
self.fileResource = Resource(self.idevice, resourceFile)
def fix_nodeResourceNotInResources(self, package, idevice, checksum, resource):
self.fix_nodeResourceNonExistant(idevice, resource)
if resource.checksum in package.resources:
package.resources[resource.checksum].remove(resource)
if not package.resources[resource.checksum]:
package.resources.pop(resource.checksum)
if resource._idevice:
if isinstance(resource._idevice.parentNode, Node):
Resource(idevice, resource.path)
else:
log.info('Not adding zombie resource %s') % resource.storageName
def set_footerImg(self, value):
"""Set the footer image for this package"""
if self._footerImg:
self._footerImg.delete()
if value:
if value.startswith("file://"):
value = value[7:]
imgFile = Path(value)
self._footerImg = Resource(self, Path(imgFile))
else:
self._footerImg = u''
def uploadFile(self, filePath):
"""
Store the upload files in the package
Needs to be in a package to work.
"""
if self.type == "descartes" and not filePath.endswith(".jar"):
if filePath.find(",") == -1:
global SCENE_NUM
SCENE_NUM = 1
else:
SCENE_NUM = int(filePath[:filePath.find(",")])
if self.type == "descartes" and (filePath.endswith(".htm")
or filePath.endswith(".html")):
global url
url = filePath
self.appletCode = self.getAppletcodeDescartes(filePath)
# none scene was found:
if self.appletCode == '':
return None
else:
log.debug(u"uploadFile " + unicode(filePath))
resourceFile = Path(filePath)
assert self.parentNode, _('file %s has no parentNode') % self.id
assert self.parentNode.package, \
_('iDevice %s has no package') % self.parentNode.id
if resourceFile.isfile():
self.message = ""
Resource(self, resourceFile)
if self.type == "geogebra":
self.appletCode = self.getAppletcodeGeogebra(
resourceFile.basename().replace(' ', '_').replace(
')', '').replace('(', ''))
if self.type == "jclic":
self.appletCode = self.getAppletcodeJClic(
resourceFile.basename().replace(' ', '_').replace(
')', '').replace('(', ''))
if self.type == "scratch":
self.appletCode = self.getAppletcodeScratch(
resourceFile.basename().replace(' ', '_').replace(
')', '').replace('(', ''))
if self.type == "descartes":
self.appletCode = self.getAppletcodeDescartes(
resourceFile.basename())
## next code should be used to load in the editor the HTML code of the html file:
# if self.type == "other":
# if filePath.endswith(".html") or filePath.endswith(".htm"):
# content = open(filePath, 'r')
# str = content.read()
# self.appletCode = str
# content.close()
# else:
# log.error('File %s is not a HTML file' % resourceFile)
else:
log.error('File %s is not a file' % resourceFile)
def prepareTestPackage(self):
idevice = FreeTextIdevice()
#idevice.setParentNode(self.package.root)
self.package.root.addIdevice(idevice)
img = Path('testing/oliver.jpg')
resource = Resource(idevice, img)
idevice.content.content = '<p><img src=\"resources/oliver.jpg\"/></p>'
idevice.content.content_w_resourcePaths = idevice.content.content
assert img.md5 in self.package.resources
assert resource in idevice.userResources
return (resource, img, idevice)
def check(self):
if self.clear:
log.info('Clearing resource references')
self.clear_resourceReferences()
log.info('Adding resource references')
for path in self.package.resourceDir.files():
if path in self.idevices:
for idevice in self.idevices[path]:
resource = Resource(idevice, path)
if isinstance(idevice, FieldWithResources):
galleryimage = GalleryImage(idevice,
'',
None,
mkThumbnail=False)
galleryimage._imageResource = resource
if isinstance(
idevice, Idevice
) and idevice.klass == 'ImageMagnifierIdevice':
idevice.imageMagnifier.imageResource = resource
if isinstance(
idevice,
Idevice) and idevice.klass == 'GalleryIdevice':
for image in idevice.images:
if image._imageResource.storageName == resource.storageName:
image._imageResource = resource
break
elif image._thumbnailResource.storageName == resource.storageName:
image._thumbnailResource = resource
break
elif self.package._backgroundImg and path == self.package._backgroundImg.path:
self.package._backgroundImg = Resource(self.package, path)
for check in dir(self):
if check.startswith('check_'):
fn = getattr(self, check)
log.info('Checking %s' % check[6:])
fn()
return self.inconsistencies
def addGameScript(self):
from exe import globals
import os,sys
scriptSrcFilename = globals.application.config.webDir/"templates"/"hangman.js"
log.debug("Script filename = " + scriptSrcFilename)
#assert(self.parentNode,
# 'Image '+self.id+' has no parentNode')
#assert(self.parentNode.package,
# 'iDevice '+self.parentNode.id+' has no package')
gameScriptFile = Path(scriptSrcFilename)
if gameScriptFile.isfile():
Resource(self, gameScriptFile)
def uploadFile(self, filePath):
"""
Store the upload files in the package
Needs to be in a package to work.
"""
log.debug(u"uploadFile "+unicode(filePath))
resourceFile = Path(filePath)
assert(self.parentNode, _('file %s has no parentNode') % self.id)
assert(self.parentNode.package, _('iDevice %s has no package') % self.parentNode.id)
if resourceFile.isfile():
self.message = ""
Resource(self, resourceFile)
if self.type == "geogebra":
self.appletCode = self.getAppletcode(resourceFile.basename())
else:
log.error('File %s is not a file' % resourceFile)
def testReferenceCounting(self):
"""
Make 3 resources with different names but same md5, ensure they are reference counted properly
"""
res1 = Path('my.resource1.bin')
res2 = Path('my.resource2.bin')
res3 = Path('my.resource3.bin')
for res in (res1, res2, res3):
res.write_bytes('SOME NICE DATA')
res1md5 = res1.md5
res4 = Path('tmp/my.resource1.bin')
if not res4.dirname().exists():
res4.dirname().makedirs()
res4.write_bytes('SOME *DIFFERENT* DATA')
res4md5 = res4.md5
res1, res2, res3, res4 = map(lambda f: Resource(self.package, f),
(res1, res2, res3, res4))
assert res1.storageName == 'my.resource1.bin', res1.storageName
assert res2.storageName == 'my.resource1.bin', res2.storageName
assert res3.storageName == 'my.resource1.bin', res3.storageName
assert res4.storageName == 'my.resource1.1.bin', res4.storageName
assert res4.userName == 'my.resource1.bin', res4.userName
assert len(self.package.resources) == 2
assert len(self.package.resources[res1.path.md5]) == 3
assert len(self.package.resources[res4.path.md5]) == 1
assert self.package.resources[res4md5][0] is res4
# Now start deleting things
res4path = Path(res4.path)
assert res4path.exists()
res4.delete()
assert not res4path.exists()
assert res4md5 not in self.package.resources
assert not res4path.exists()
res1path = Path(res1.path)
assert res1path.exists()
res2.delete()
assert res1path.exists()
assert res2 not in self.package.resources[res1md5]
res1.delete()
assert res1path.exists()
assert res1 not in self.package.resources[res1md5]
res3.delete()
assert res1md5 not in self.package.resources
assert not res1path.exists()
def uploadFile(self, filePath):
if self.fileResource is not None:
self.fileResource.delete()
finalName = str(filePath)
if self.alwaysNameTo is not None:
from os.path import dirname
from shutil import copyfile
dirName = dirname(filePath)
finalName = dirName + "/" + self.alwaysNameTo
copyfile(filePath, finalName)
if self.fileDescription.content == "":
self.fileDescription.content = os.path.basename(filePath)
resourceFile = Path(finalName)
if resourceFile.isfile():
self.idevice.message = ""
self.fileResource = Resource(self.idevice, resourceFile)
def setAttachment(self, attachmentPath):
"""
Store the attachment in the package
Needs to be in a package to work.
"""
log.debug(u"setAttachment " + unicode(attachmentPath))
resourceFile = Path(attachmentPath)
assert self.parentNode, \
_('Attachment %s has no parentNode') % self.id
assert self.parentNode.package, \
_('iDevice %s has no package') % self.parentNode.id
if resourceFile.isfile():
if self.userResources:
# Clear out old attachment/s
while self.userResources:
self.userResources[0].delete()
# Create the new resource
Resource(self, resourceFile)
else:
log.error('File %s is not a file' % resourceFile)
def uploadFile(self, filePath):
"""
Store the upload files in the package
Needs to be in a package to work.
"""
if self.type == "descartes" and not filePath.endswith(".jar"):
if filePath.find(",") == -1:
global SCENE_NUM
SCENE_NUM = 1
else:
SCENE_NUM = int(filePath[:filePath.find(",")])
if (filePath.endswith(".htm") or filePath.endswith(".html")):
global url
url = filePath
self.appletCode = self.getAppletcodeDescartes(filePath)
# none scene was found:
if self.appletCode == '':
return None
else:
log.debug(u"uploadFile "+unicode(filePath))
resourceFile = Path(filePath)
assert self.parentNode, _('file %s has no parentNode') % self.id
assert self.parentNode.package, \
_('iDevice %s has no package') % self.parentNode.id
if resourceFile.isfile():
self.message = ""
Resource(self, resourceFile)
if self.type == "geogebra":
self.appletCode = self.getAppletcodeGeogebra(resourceFile.basename())
if self.type == "jclic":
self.appletCode = self.getAppletcodeJClic(resourceFile.basename())
if self.type == "scratch":
self.appletCode = self.getAppletcodeScratch(resourceFile.basename())
if self.type == "descartes":
self.appletCode = self.getAppletcodeDescartes(resourceFile.basename())
else:
log.error('File %s is not a file' % resourceFile)
class WikipediaIdevice(Idevice):
"""
A Wikipedia Idevice is one built from a Wikipedia article.
"""
persistenceVersion = 9
def __init__(self, defaultSite):
Idevice.__init__(
self, x_(u"Wiki Article"), x_(u"University of Auckland"),
x_(u"""<p>The Wikipedia iDevice allows you to locate
existing content from within Wikipedia and download this content into your eXe
resource. The Wikipedia Article iDevice takes a snapshot copy of the article
content. Changes in Wikipedia will not automatically update individual snapshot
copies in eXe, a fresh copy of the article will need to be taken. Likewise,
changes made in eXe will not be updated in Wikipedia. </p> <p>Wikipedia content
is covered by the GNU Free Documentation 1.2 License, and since 2009 additionally
by the Creative Commons Attribution-ShareAlike 3.0 Unported License.</p>"""),
u"", u"")
self.emphasis = Idevice.NoEmphasis
self.articleName = u""
self.article = TextAreaField(x_(u"Article"))
self.article.idevice = self
self.images = {}
self.site = defaultSite
self.icon = u"inter"
self._langInstruc = x_(u"""Select the appropriate language version
of Wikipedia to search and enter search term.""")
self._searchInstruc = x_("""Enter a phrase or term you wish to search
within Wikipedia.""")
self.ownUrl = ""
self.systemResources += ['exe_wikipedia.css']
# Properties
langInstruc = lateTranslate('langInstruc')
searchInstruc = lateTranslate('searchInstruc')
def loadArticle(self, name):
"""
Load the article from Wikipedia
"""
self.articleName = name
url = ''
name = urllib.quote(name.replace(" ", "_").encode('utf-8'))
# Get the full URL of the site
url = self.site or self.ownUrl
if not url.endswith('/') and name != '':
url += '/'
if '://' not in url:
url = 'http://' + url
url += name
# Get the site content
try:
net = urllib.urlopen(url)
page = net.read()
net.close()
except IOError, error:
log.warning(unicode(error))
self.article.content = _(
u"Unable to download from %s <br/>Please check the spelling and connection and try again."
) % url
self.article.content_w_resourcePaths = self.article.content
self.article.content_wo_resourcePaths = self.article.content
return
page = unicode(page, "utf8")
# FIXME avoid problems with numeric entities in attributes
page = page.replace(u' ', u' ')
# avoidParserProblems is set to False because BeautifulSoup's
# cleanup was causing a "concatenating Null+Str" error,
# and Wikipedia's HTML doesn't need cleaning up.
# BeautifulSoup is faster this way too.
soup = BeautifulSoup(page, False)
content = soup.first('div', {'id': "content"})
# Fix bug #1359: El estilo ITE no respeta ancho de página al exportar
# a páginas web si se usa iDevice wikipedia
content['id'] = "wikipedia-content"
# remove the wiktionary, wikimedia commons, and categories boxes
# and the protected icon and the needs citations box
if content:
infoboxes = content.findAll('div',
{'class': 'infobox sisterproject'})
[infobox.extract() for infobox in infoboxes]
catboxes = content.findAll('div', {'id': 'catlinks'})
[catbox.extract() for catbox in catboxes]
amboxes = content.findAll('table',
{'class': re.compile(r'.*\bambox\b.*')})
[ambox.extract() for ambox in amboxes]
protecteds = content.findAll('div', {'id': 'protected-icon'})
[protected.extract() for protected in protecteds]
# Extract HTML comments
comments = content.findAll(
text=lambda text: isinstance(text, Comment))
[comment.extract() for comment in comments]
else:
content = soup.first('body')
# If we still don't have content it means there is a problem with the article
if not content:
log.error(u"No content on Wikipedia article: %s" % url)
self.article.content = _(
u"Unable to download from %s <br/>Please check the spelling and connection and try again."
) % url
# Set the other elements as well
self.article.content_w_resourcePaths = self.article.content
self.article.content_wo_resourcePaths = self.article.content
return
# Clear out any old images
while self.userResources:
self.userResources[0].delete()
self.images = {}
# Download the images
bits = url.split('/')
netloc = '%s//%s' % (bits[0], bits[2])
path = '/'.join(bits[3:-1])
tmpDir = TempDirPath()
# Fetch all images from content
for imageTag in content.fetch('img'):
# Get src and image filename
imageSrc = unicode(imageTag['src'])
imageName = imageSrc.split('/')[-1]
imageName = imageName.replace('>', '>')
imageName = imageName.replace('<', '<')
imageName = imageName.replace('"', '"')
imageName = imageName.replace(' ', '')
imageName = imageName.replace('%2C', ',')
imageName = imageName.replace('%22', '"')
imageName = imageName.replace('%28', '(')
imageName = imageName.replace('%29', ')')
imageName = imageName.replace('%C3%A5', 'å')
# Decode image name
imageName = urllib.unquote(imageName)
# Search if we've already got this image
if imageName not in self.images:
if not re.match("^http(s)?:\/\/", imageSrc):
if imageSrc.startswith("/"):
imageSrc = bits[0] + imageSrc
else:
imageSrc = '%s/%s/%s' % (netloc, path, imageSrc)
try:
# download with its original name... in ASCII:
## er... just because some repositories do not undestand no ascii names of files:
imageName = imageName.encode('ascii', 'ignore')
# If the image file doesn't have an extension try to guess it
if not re.match('^.*\.(.){1,}', imageName):
# Open a conecction with the image and get the headers
online_resource = urllib.urlopen(imageSrc)
image_info = online_resource.info()
online_resource.close()
# Try to guess extension from mimetype
extension = mimetypes.guess_extension(
image_info['content-type'])
# Wikimedia uses mainly SVG images so we can safely say that
# this image is in svg (if it wasn't if wouldn't be shown anyway)
extension = extenion or '.svg'
imageName = imageName + extension
# Download image
urllib.urlretrieve(imageSrc, tmpDir / imageName)
# Add the new resource
new_resource = Resource(self, tmpDir / imageName)
except:
log.error(u'Unable to download file: %s' % imageSrc)
# If there is an exception try to get the next one
continue
if new_resource._storageName != imageName:
# looks like it was changed due to a possible conflict,
# so reset the imageName accordingly for the content:
imageName = new_resource._storageName
self.images[imageName] = True
imageTag['src'] = u"resources/" + imageName
self.article.content = self.reformatArticle(netloc, unicode(content))
# now that these are supporting images, any direct manipulation
# of the content field must also store this updated information
# into the other corresponding fields of TextAreaField:
# (perhaps eventually a property should be made for TextAreaField
# such that these extra set's are not necessary, but for now, here:)
self.article.content_w_resourcePaths = self.article.content
self.article.content_wo_resourcePaths = self.article.content
class WikipediaIdevice(Idevice):
"""
A Wikipedia Idevice is one built from a Wikipedia article.
"""
persistenceVersion = 8
def __init__(self, defaultSite):
Idevice.__init__(self, x_(u"Wiki Article"),
x_(u"University of Auckland"),
x_(u"""<p>The Wikipedia iDevice allows you to locate
existing content from within Wikipedia and download this content into your eXe
resource. The Wikipedia Article iDevice takes a snapshot copy of the article
content. Changes in Wikipedia will not automatically update individual snapshot
copies in eXe, a fresh copy of the article will need to be taken. Likewise,
changes made in eXe will not be updated in Wikipedia. </p> <p>Wikipedia content
is covered by the GNU free documentation license.</p>"""),
u"", u"")
self.emphasis = Idevice.NoEmphasis
self.articleName = u""
self.article = TextAreaField(x_(u"Article"))
self.article.idevice = self
self.images = {}
self.site = defaultSite
self.icon = u"inter"
self.systemResources += ["fdl.html"]
self._langInstruc = x_(u"""Select the appropriate language version
of Wikipedia to search and enter search term.""")
self._searchInstruc = x_("""Enter a phrase or term you wish to search
within Wikipedia.""")
self.ownUrl = ""
# Properties
langInstruc = lateTranslate('langInstruc')
searchInstruc = lateTranslate('searchInstruc')
def loadArticle(self, name):
"""
Load the article from Wikipedia
"""
self.articleName = name
url = ""
name = urllib.quote(name.replace(" ", "_").encode('utf-8'))
try:
url = (self.site or self.ownUrl)
if not url.endswith('/') and name <> '': url += '/'
if '://' not in url: url = 'http://' + url
url += name
net = urllib.urlopen(url)
page = net.read()
net.close()
except IOError, error:
log.warning(unicode(error))
self.article.content = _(u"Unable to download from %s <br/>Please check the spelling and connection and try again.") % url
self.article.content_w_resourcePaths = self.article.content
self.article.content_wo_resourcePaths = self.article.content
return
page = unicode(page, "utf8")
# FIXME avoid problems with numeric entities in attributes
page = page.replace(u' ', u' ')
# avoidParserProblems is set to False because BeautifulSoup's
# cleanup was causing a "concatenating Null+Str" error,
# and Wikipedia's HTML doesn't need cleaning up.
# BeautifulSoup is faster this way too.
soup = BeautifulSoup(page, False)
content = soup.first('div', {'id': "content"})
# remove the wiktionary, wikimedia commons, and categories boxes
# and the protected icon and the needs citations box
if content:
infoboxes = content.findAll('div',
{'class' : 'infobox sisterproject'})
[infobox.extract() for infobox in infoboxes]
catboxes = content.findAll('div', {'id' : 'catlinks'})
[catbox.extract() for catbox in catboxes]
amboxes = content.findAll('table',
{'class' : re.compile(r'.*\bambox\b.*')})
[ambox.extract() for ambox in amboxes]
protecteds = content.findAll('div', {'id' : 'protected-icon'})
[protected.extract() for protected in protecteds]
else:
content = soup.first('body')
if not content:
log.error("no content")
self.article.content = _(u"Unable to download from %s <br/>Please check the spelling and connection and try again.") % url
# set the other elements as well
self.article.content_w_resourcePaths = self.article.content
self.article.content_wo_resourcePaths = self.article.content
return
# clear out any old images
while self.userResources:
self.userResources[0].delete()
self.images = {}
# Download the images
bits = url.split('/')
netloc = '%s//%s' % (bits[0], bits[2])
path = '/'.join(bits[3:-1])
tmpDir = TempDirPath()
for imageTag in content.fetch('img'):
imageSrc = unicode(imageTag['src'])
imageName = imageSrc.split('/')[-1]
# Search if we've already got this image
if imageName not in self.images:
if not imageSrc.startswith("http://"):
if imageSrc.startswith("/"):
imageSrc = netloc + imageSrc
else:
imageSrc = '%s/%s/%s' % (netloc, path, imageSrc)
urllib.urlretrieve(imageSrc, tmpDir/imageName)
new_resource = Resource(self, tmpDir/imageName)
if new_resource._storageName != imageName:
# looks like it was changed due to a possible conflict,
# so reset the imageName accordingly for the content:
imageName = new_resource._storageName
self.images[imageName] = True
# We have to use absolute URLs if we want the images to
# show up in edit mode inside FCKEditor
imageTag['src'] = (u"/" + self.parentNode.package.name + u"/resources/" + imageName)
self.article.content = self.reformatArticle(netloc, unicode(content))
# now that these are supporting images, any direct manipulation
# of the content field must also store this updated information
# into the other corresponding fields of TextAreaField:
# (perhaps eventually a property should be made for TextAreaField
# such that these extra set's are not necessary, but for now, here:)
self.article.content_w_resourcePaths = self.article.content
self.article.content_wo_resourcePaths = self.article.content
class FileField(Field):
"""
Field for storing an individual file
"""
persistenceVersion = 5
"""
alwaysNameTo - make sure that this file always a certain final name
"""
def __init__(self, idevice, alwaysNameTo=None, desc="File Field", help="File Field Help"):
Field.__init__(self, desc, help)
self.idevice = idevice
self.fileResource = None
self.fileInstruc = "Upload a file"
self.alwaysNameTo = alwaysNameTo
self.fileDescription = TextField("Description")
self.fileDescription.idevice = self
def uploadFile(self, filePath):
if self.fileResource is not None:
self.fileResource.delete()
finalName = str(filePath)
if self.alwaysNameTo is not None:
from os.path import dirname
from shutil import copyfile
dirName = dirname(filePath)
finalName = dirName + "/" + self.alwaysNameTo
copyfile(filePath, finalName)
if self.fileDescription.content == "":
self.fileDescription.content = os.path.basename(filePath)
resourceFile = Path(finalName)
if resourceFile.isfile():
self.idevice.message = ""
self.fileResource = Resource(self.idevice, resourceFile)
def deleteFile(self):
if self.fileResource is not None:
self.fileResource.delete()
self.fileResource = None
def get_translatable_properties(self):
"""
Get a list of translatable property names that can be translated.
:rtype: list
:return: List of translatable properties of the field.
"""
# The only translatable fields in this Idevice is the description (which
# is a field in itself)
return self.fileDescription.get_translatable_properties()
def translate(self):
"""
Do the actual translation.
"""
# We only have a translatable property, as it is also a field, just call
# translate() on it
self.fileDescription.translate()
def upgradeToVersion4(self):
self.fileDescription = TextField("Description")
def upgradeToVersion5(self):
self.fileDescription.idevice = self
class FileField(Field):
"""
Field for storing an individual file
"""
persistenceVersion = 5
"""
alwaysNameTo - make sure that this file always a certain final name
"""
def __init__(self,
idevice,
alwaysNameTo=None,
desc="File Field",
help="File Field Help"):
Field.__init__(self, desc, help)
self.idevice = idevice
self.fileResource = None
self.fileInstruc = "Upload a file"
self.alwaysNameTo = alwaysNameTo
self.fileDescription = TextField("Description")
self.fileDescription.idevice = self
def uploadFile(self, filePath):
if self.fileResource is not None:
self.fileResource.delete()
finalName = str(filePath)
if self.alwaysNameTo is not None:
from os.path import dirname
from shutil import copyfile
dirName = dirname(filePath)
finalName = dirName + "/" + self.alwaysNameTo
copyfile(filePath, finalName)
if self.fileDescription.content == "":
self.fileDescription.content = os.path.basename(filePath)
resourceFile = Path(finalName)
if resourceFile.isfile():
self.idevice.message = ""
self.fileResource = Resource(self.idevice, resourceFile)
def deleteFile(self):
if self.fileResource is not None:
self.fileResource.delete()
self.fileResource = None
def get_translatable_properties(self):
"""
Get a list of translatable property names that can be translated.
:rtype: list
:return: List of translatable properties of the field.
"""
# The only translatable fields in this Idevice is the description (which
# is a field in itself)
return self.fileDescription.get_translatable_properties()
def translate(self):
"""
Do the actual translation.
"""
# We only have a translatable property, as it is also a field, just call
# translate() on it
self.fileDescription.translate()
def upgradeToVersion4(self):
self.fileDescription = TextField("Description")
def upgradeToVersion5(self):
self.fileDescription.idevice = self
def _insertNode(self, node, url, depth=0, idevice=None):
if self.cancel:
return
if isinstance(url, str):
link = None
url = self.resources['urls'][url]
elif isinstance(url, Link):
link = url
url = link.url
if url.mime == 'text/html' and self.depths[str(url)] >= depth:
if self.client:
self.client.call(
'eXe.app.getController("Toolbar").updateImportProgressWindow',
_(u'Inserting %s') % (str(url)))
type = link.tag.name if link and link.tag else 'a'
if type not in ['frame', 'iframe'] and node:
node = node.createChild()
node.setTitle(self._guessName(url))
if depth == 1:
node.up()
if not node:
node = self.node
parent = idevice if type in ['frame', 'iframe'] else None
idevice = FreeTextIdevice(type=type, parent=parent)
idevice.edit = False
node.addIdevice(idevice)
if url.type == "file":
p = Path(self.baseurl + os.path.sep + str(url))
p.setSalt(str(url))
r = Resource(idevice, p)
url.storageName = quote(r.storageName)
if link and link.relative not in link.referrer.contentUpdated:
if link.match:
link.referrer.content = link.referrer.content.replace(
link.match, '###resources###/%s' % (url.storageName))
else:
link.referrer.content = link.referrer.content.replace(
link.relative,
'###resources###/%s' % (url.storageName))
link.referrer.contentUpdated.append(link.relative)
if self.depths[str(url)] < depth:
return
for l in url.links:
if self.cancel:
return
self._insertNode(node, l, depth + 1, idevice)
content = url.getContent()
if content:
content_w_resourcePaths = re.sub('###resources###/', 'resources/',
content)
content_wo_resourcePaths = re.sub('###resources###/', '', content)
if url.mime == "text/html" and idevice:
soup = url.getSoup()
if soup and soup.declaredHTMLEncoding:
content_w_resourcePaths = re.sub(soup.declaredHTMLEncoding,
'utf-8',
content_w_resourcePaths,
re.IGNORECASE)
content_wo_resourcePaths = re.sub(
soup.declaredHTMLEncoding, 'utf-8',
content_wo_resourcePaths, re.IGNORECASE)
if soup and soup.find('frameset'):
idevice.type = 'frameset'
idevice.setContent(content_w_resourcePaths,
content_wo_resourcePaths)
f = open(r.path, "w")
f.write(content_wo_resourcePaths.encode('utf-8'))
f.close()
while self.userResources:
self.userResources[0].delete()
self.images = {}
# Download the images
imgpath = os.path.dirname(path)
for imageTag in content.fetch('img'):
imageSrc = unicode(os.path.join(tempdir, "images", \
imageTag['src'].split('/')[-1]))
imageName = os.path.basename(imageSrc)
# Search if we've already got this image
if not os.path.exists(imageSrc):
log.error("Image file %s not found" % imageName)
elif imageName not in self.images:
new_image = Path(imageSrc)
new_resource = Resource(self, new_image)
if new_resource._storageName != imageName:
# looks like it was changed due to a possible conflict,
# so reset the imageName accordingly for the content:
imageName = new_resource._storageName
self.images[imageName] = True
# We have to use absolute URLs if we want the images to
# show up in edit mode inside FCKEditor
imageTag['src'] = (u"/" + self.parentNode.package.name +
u"/resources/" + imageName)
self.article.content = self.reformatArticle('/', unicode(content))
# now that these are supporting images, any direct manipulation
# of the content field must also store this updated information
# into the other corresponding fields of TextAreaField:
# (perhaps eventually a property should be made for TextAreaField
# such that these extra set's are not necessary, but for now, here:)
