def _listPath(cls, rootPath, recursive, **kwargs):
listDirs = ArgsUtils.get('listDirs', False, kwargs)
skipSVN = ArgsUtils.get('skipSVN', True, kwargs)
skips = ArgsUtils.get('skips', None, kwargs)
allowExtensions = ArgsUtils.getAsList('allowExtensions', kwargs)
skipExtensions = ArgsUtils.getAsList('skipExtensions', kwargs)
out = []
for item in os.listdir(rootPath):
if (skipSVN and item == '.svn') or (skips and item in skips):
continue
absItem = os.path.join(rootPath, item)
if os.path.isdir(absItem):
path = (absItem + os.sep)
if listDirs:
out.append(path)
absItem = None
if recursive:
out += cls._listPath(path, recursive, **kwargs)
elif os.path.isfile(absItem):
if skipExtensions and StringUtils.ends(item, skipExtensions):
continue
if allowExtensions and not StringUtils.ends(item, allowExtensions):
continue
if absItem:
out.append(absItem)
return out
def _copyWalker(self, walkData):
staticFolder = False
for folder in self._staticPaths:
path = FileUtils.cleanupPath(walkData.folder, isDir=True)
folder = FileUtils.cleanupPath(folder, isDir=True)
if path == folder or FileUtils.isInFolder(path, folder):
staticFolder = True
break
copiedNames = []
for item in walkData.names:
if not staticFolder and not StringUtils.ends(item, self._FILE_COPY_TYPES):
continue
sourcePath = FileUtils.createPath(walkData.folder, item)
if os.path.isdir(sourcePath):
continue
destPath = FileUtils.changePathRoot(
sourcePath, self.sourceWebRootPath, self.targetWebRootPath)
try:
FileUtils.getDirectoryOf(destPath, createIfMissing=True)
shutil.copy(sourcePath, destPath)
lastModified = FileUtils.getUTCModifiedDatetime(sourcePath)
SiteProcessUtils.createHeaderFile(destPath, lastModified)
SiteProcessUtils.copyToCdnFolder(destPath, self, lastModified)
copiedNames.append(item)
except Exception, err:
self.writeLogError(u'Unable to copy file', error=err, extras={
'SOURCE':sourcePath,
'TARGET':destPath })
return
def path(self):
if not self._path:
return None
if not StringUtils.ends(self._path, '.csv'):
if not FileUtils.getFileExtension(self._path):
self._path += '.csv'
return FileUtils.cleanupPath(self._path, isFile=True)
def _deployWalker(self, args, path, names):
"""Doc..."""
# Skip CDN file uploads when not walking the CDN root path explicitly
if not args['cdn'] and path.find(StaticFlowEnvironment.CDN_ROOT_PREFIX) != -1:
return
for name in names:
namePath = FileUtils.createPath(path, name)
if os.path.isdir(namePath) or StringUtils.ends(name, self._SKIP_EXTENSIONS):
continue
headersPath = namePath + '.headers'
if os.path.exists(headersPath):
headers = JSON.fromFile(headersPath)
else:
headers = dict()
if self._forceAll:
lastModified = None
elif self._forceHtml and StringUtils.ends(name, self._FORCE_HTML_EXTENSIONS):
lastModified = None
else:
lastModified = ArgsUtils.extract('_LAST_MODIFIED', None, headers)
if lastModified:
lastModified = TimeUtils.webTimestampToDateTime(lastModified)
kwargs = dict(
key=u'/' + namePath[len(self._localRootPath):].replace(u'\\', u'/').strip(u'/'),
maxAge=headers.get('max-age', -1),
eTag=headers.get('eTag', None),
expires=headers.get('Expires'),
newerThanDate=lastModified,
policy=S3Bucket.PUBLIC_READ)
if StringUtils.ends(name, self._STRING_EXTENSIONS):
result = self._bucket.put(
contents=FileUtils.getContents(namePath),
zipContents=True,
**kwargs)
else:
result = self._bucket.putFile(filename=namePath, **kwargs)
if result:
self._logger.write(u'DEPLOYED: ' + unicode(namePath) + u'->' + unicode(kwargs['key']))
def _createIcon(self, binPath):
iconPath = self._getIconPath()
if not iconPath:
return iconPath
if os.path.isfile(iconPath):
return iconPath
#-------------------------------------------------------------------------------------------
# MAC ICON CREATION
# On OSX use Apple's iconutil (XCode developer tools must be installed) to create an
# icns file from the icons.iconset folder at the specified location.
if OsUtils.isMac():
targetPath = FileUtils.createPath(binPath, self.appDisplayName + '.icns', isFile=True)
result = SystemUtils.executeCommand([
'iconutil', '-c', 'icns', '-o', '"' + targetPath + '"', '"' + iconPath + '"'])
if result['code']:
return ''
return targetPath
#-------------------------------------------------------------------------------------------
# WINDOWS ICON CREATION
# On Windows use convert (ImageMagick must be installed and on the PATH) to create an
# ico file from the icons folder of png files.
result = SystemUtils.executeCommand('where convert')
if result['code']:
return ''
items = result['out'].replace('\r', '').strip().split('\n')
convertCommand = None
for item in items:
if item.find('System32') == -1:
convertCommand = item
break
if not convertCommand:
return ''
images = os.listdir(iconPath)
cmd = ['"' + convertCommand + '"']
for image in images:
if not StringUtils.ends(image, ('.png', '.jpg')):
continue
imagePath = FileUtils.createPath(iconPath, image, isFile=True)
cmd.append('"' + imagePath + '"')
if len(cmd) < 2:
return ''
targetPath = FileUtils.createPath(binPath, self.appDisplayName + '.ico', isFile=True)
cmd.append('"' + targetPath + '"')
result = SystemUtils.executeCommand(cmd)
if result['code'] or not os.path.exists(targetPath):
print 'FAILED:'
print result['command']
print result['error']
return ''
return targetPath
def _cleanupInFolder(self, arg, dirname, names):
for name in names:
if StringUtils.ends(name, self._compiler.ignoreExtensions):
os.remove(FileUtils.createPath(dirname, name, isFile=True))
# Deletes python (.py) files associated with ui files so only .pyc files remain.
if name.endswith('.ui'):
pyName = name.rsplit('.', 1)[0] + '.py'
pyNamePath = FileUtils.createPath(dirname, pyName, isFile=True)
if os.path.exists(pyNamePath):
os.remove(pyNamePath)
def testFileFilter(cls, path, extensionFilter =None, nameFilter =None):
# Skip extensions not included in the filter if the filter exists
if extensionFilter and not StringUtils.ends(path, extensionFilter):
print 'FOLDER[skipped extension]:', path
return False
# Skip names not included in the filter if the filter exists
if nameFilter and isinstance(nameFilter, basestring):
nameFilter = re.compile(nameFilter)
if nameFilter and not nameFilter.search(os.path.basename(path)):
print 'FOLDER[skipped name]:', path
return False
return True
def getFileExtension(cls, path):
""" Retrieves the file extension for the specified path if an
extension exists. If no file extension exists, or the path is not
a file type, the method returns none.
"""
if not path:
return None
if StringUtils.ends(path, os.path.sep):
return None
filename = path.split(os.path.sep)[-1]
if not filename:
return None
parts = filename.split(os.path.extsep)
if len(filename) < 2:
return None
return filename[-1]
def getFileExtension(cls, path):
""" Retrieves the file extension for the specified path if an
extension exists. If no file extension exists, or the path is not
a file type, the method returns none.
"""
if not path:
return None
if StringUtils.ends(path, os.path.sep):
return None
filename = path.split(os.path.sep)[-1]
if not filename:
return None
parts = filename.split(os.path.extsep)
if len(filename) < 2:
return None
return filename[-1]
def _processFolderDefinitions(self, path):
cd = ConfigsDict(JSON.fromFile(path))
directory = FileUtils.getDirectoryOf(path)
for item in os.listdir(directory):
# Only find content source file types
if not StringUtils.ends(item, ('.sfml', '.html')):
continue
# Skip files that already have a definitions file
itemPath = FileUtils.createPath(directory, item, isFile=True)
itemDefsPath = itemPath.rsplit('.', 1)[0] + '.def'
if os.path.exists(itemDefsPath):
continue
test = SiteProcessUtils.testFileFilter(
itemPath,
cd.get(('FOLDER', 'EXTENSION_FILTER')),
cd.get(('FOLDER', 'NAME_FILTER')))
if not test:
continue
JSON.toFile(itemDefsPath, dict(), pretty=True)
return True
def _setColor(self, color, colorMode =None):
if color is None:
return False
if isinstance(color, list):
normalized = colorMode.endswith('one')
if colorMode.startswith('rgb'):
keys = ColorTag._RGB_KEYS
elif colorMode.startswith('hsl'):
keys = ColorTag._HSL_KEYS
elif colorMode.startswith('hsv'):
keys = ColorTag._HSV_KEYS
else:
keys = ColorTag._RGB_KEYS
c = {}
norms = True
for i in range(len(color)):
raw = color[i]
value = float(raw.rstrip('+-'))
c[keys[i]] = value
if not normalized:
norms = norms and (raw.find('.') != -1 and value <= 1.0)
c = ColorValue(c, normalized or norms)
if StringUtils.ends(color[-1], ['+', '-']):
c.bend(color[-1])
else:
c = color
self.attrs.styles.add(
'color', c.web if isinstance(c, ColorValue) else unicode(c), self.attrs.styleGroup
)
return True
def _listPath(cls, rootPath, recursive, **kwargs):
allowDots = kwargs.get('allowDots', True)
rootPath = cls.cleanupPath(rootPath, isDir=True)
listFiles = kwargs.get('listFiles', True)
listDirs = kwargs.get('listDirs', False)
skipSVN = kwargs.get('skipSVN', True)
skips = kwargs.get('skips', None)
absolute = kwargs.get('absolute', True)
pieces = kwargs.get('pieces', False)
topPath = ArgsUtils.extract('topPath', rootPath, kwargs)
allowExtensions = ArgsUtils.getAsList('allowExtensions', kwargs)
skipExtensions = ArgsUtils.getAsList('skipExtensions', kwargs)
out = []
for item in os.listdir(rootPath):
if not allowDots and item.startswith('.'):
continue
if (skipSVN and item == '.svn') or (skips and item in skips):
continue
absItem = os.path.join(rootPath, item)
if os.path.isdir(absItem):
path = absItem + os.sep
if listDirs:
out.append(path if absolute else item)
absItem = None
if recursive:
out += cls._listPath(rootPath=path,
recursive=recursive,
topPath=topPath,
**kwargs)
elif os.path.isfile(absItem):
skip = skipExtensions and StringUtils.ends(
item, skipExtensions)
if not listFiles or skip:
continue
if allowExtensions and not StringUtils.ends(
item, allowExtensions):
continue
if not absItem:
continue
if not pieces and not absolute:
out.append(item)
continue
relativeItem = absItem[len(topPath):].strip(os.sep).split(os.sep)
if absolute:
relativeItem.insert(0, topPath)
if pieces:
out.append(relativeItem)
else:
out.append(os.path.join(*relativeItem))
return out
def _cleanupWalker(self, args, path, names):
for name in names:
itemPath = FileUtils.createPath(path, name)
if not os.path.isfile(itemPath) or not StringUtils.ends(name, self._SKIP_EXTENSIONS):
continue
os.remove(itemPath)
def _removeHTMLTagWhitespace(self, source):
preserveBlocks = DomUtils.getPreserveBlocks(source)
res = MarkupProcessor._HTML_INSIDE_TAG_PATTERN.finditer(source)
if res:
for r in res:
start = r.start()
end = r.end()
replace = r.group('tag').replace(u'\n', u' ')
source = source[:start] + replace + source[end:]
res = MarkupProcessor._HTML_PRE_TAG_WHITESPACE_PATTERN.finditer(source)
if res:
for r in res:
start = r.start()
end = r.end()
tag = r.group('tag')
preSource = source[:start]
if StringUtils.begins(tag, (u'<span', u'<a')):
strippedPreSource = preSource.strip()
# Preserve lines between span tags
if StringUtils.ends(strippedPreSource, (u'</span>', u'</a>')):
continue
# Preserve lines between span tags and non-html entities like text
if not strippedPreSource.endswith(u'>'):
continue
skip = False
for b in preserveBlocks:
if b.start <= start <= b.end or b.start <= end <= b.end:
skip = True
break
if skip:
continue
length = len(r.group('whitespace'))
replace = u' '*length + tag
source = preSource + replace + source[end:]
res = MarkupProcessor._HTML_POST_TAG_WHITESPACE_PATTERN.finditer(source)
if res:
for r in res:
start = r.start()
end = r.end()
tag = r.group('tag')
postSource = source[end:]
if tag in (u'</span>', u'</a>'):
strippedPostSource = postSource.strip()
# Preserve lines between span tags
if StringUtils.begins(strippedPostSource, (u'<span', u'<a')):
continue
# Preserve lines between span tags and non-html entities like text
if not strippedPostSource.startswith(u'<'):
continue
skip = False
for b in preserveBlocks:
if b.start <= start <= b.end or b.start <= end <= b.end:
skip = True
break
if skip:
continue
length = len(r.group('whitespace'))
replace = tag + u' '*length
source = source[:start] + replace + postSource
return source
def _listPath(cls, rootPath, recursive, **kwargs):
allowDots = kwargs.get('allowDots', True)
rootPath = cls.cleanupPath(rootPath, isDir=True)
listFiles = kwargs.get('listFiles', True)
listDirs = kwargs.get('listDirs', False)
skipSVN = kwargs.get('skipSVN', True)
skips = kwargs.get('skips', None)
absolute = kwargs.get('absolute', True)
pieces = kwargs.get('pieces', False)
topPath = ArgsUtils.extract('topPath', rootPath, kwargs)
allowExtensions = ArgsUtils.getAsList('allowExtensions', kwargs)
skipExtensions = ArgsUtils.getAsList('skipExtensions', kwargs)
out = []
for item in os.listdir(rootPath):
if not allowDots and item.startswith('.'):
continue
if (skipSVN and item == '.svn') or (skips and item in skips):
continue
absItem = os.path.join(rootPath, item)
if os.path.isdir(absItem):
path = absItem + os.sep
if listDirs:
out.append(path if absolute else item)
absItem = None
if recursive:
out += cls._listPath(
rootPath=path,
recursive=recursive,
topPath=topPath, **kwargs)
elif os.path.isfile(absItem):
skip = skipExtensions and StringUtils.ends(item, skipExtensions)
if not listFiles or skip:
continue
if allowExtensions and not StringUtils.ends(item, allowExtensions):
continue
if not absItem:
continue
if not pieces and not absolute:
out.append(item)
continue
relativeItem = absItem[len(topPath):].strip(os.sep).split(os.sep)
if absolute:
relativeItem.insert(0, topPath)
if pieces:
out.append(relativeItem)
else:
out.append(os.path.join(*relativeItem))
return out
