def _deployResources(cls):
""" On windows the resource folder data is stored within the application install directory.
However, due to permissions issues, certain file types cannot be accessed from that
directory without causing the program to crash. Therefore, the stored resources must
be expanded into the user's AppData/Local folder. The method checks the currently
deployed resources folder and deploys the stored resources if the existing resources
either don't exist or don't match the currently installed version of the program. """
if not OsUtils.isWindows() or not PyGlassEnvironment.isDeployed:
return False
storagePath = PyGlassEnvironment.getInstallationPath('resource_storage', isDir=True)
storageStampPath = FileUtils.makeFilePath(storagePath, 'install.stamp')
resourcePath = PyGlassEnvironment.getRootResourcePath(isDir=True)
resourceStampPath = FileUtils.makeFilePath(resourcePath, 'install.stamp')
try:
resousrceData = JSON.fromFile(resourceStampPath)
storageData = JSON.fromFile(storageStampPath)
if resousrceData['CTS'] == storageData['CTS']:
return False
except Exception as err:
pass
SystemUtils.remove(resourcePath)
FileUtils.mergeCopy(storagePath, resourcePath)
return True
def run(self):
"""Doc..."""
resources = self._compiler.resources
#-------------------------------------------------------------------------------------------
# RESOURCES
# If no resource folders were specified copy the entire contents of the resources
# folder. Make sure to skip the local resources path in the process.
if not resources:
for item in os.listdir(PyGlassEnvironment.getRootResourcePath(isDir=True)):
itemPath = PyGlassEnvironment.getRootResourcePath(item)
if os.path.isdir(itemPath) and not item in ['local', 'apps']:
resources.append(item)
for container in resources:
parts = container.replace('\\', '/').split('/')
self._copyResourceFolder(
PyGlassEnvironment.getRootResourcePath(*parts, isDir=True), parts)
#-------------------------------------------------------------------------------------------
# APP RESOURCES
appResources = self._compiler.resourceAppIds
if not appResources:
appResources = []
for appResource in appResources:
itemPath = PyGlassEnvironment.getRootResourcePath('apps', appResource, isDir=True)
if not os.path.exists(itemPath):
self._log.write('[WARNING]: No such app resource path found: %s' % appResource)
continue
self._copyResourceFolder(itemPath, ['apps', appResource])
#-------------------------------------------------------------------------------------------
# PYGLASS RESOURCES
# Copy the resources from the PyGlass
resources = []
for item in os.listdir(PyGlassEnvironment.getPyGlassResourcePath('..', isDir=True)):
itemPath = PyGlassEnvironment.getPyGlassResourcePath('..', item)
if os.path.isdir(itemPath):
resources.append(item)
for container in resources:
self._copyResourceFolder(
PyGlassEnvironment.getPyGlassResourcePath('..', container), [container])
# Create a stamp file in resources for comparing on future installations
creationStampFile = FileUtils.makeFilePath(self._targetPath, 'install.stamp')
JSON.toFile(creationStampFile, {'CTS':TimeUtils.toZuluPreciseTimestamp()})
#-------------------------------------------------------------------------------------------
# CLEANUP
if self._verbose:
self._log.write('CLEANUP: Removing unwanted destination files.')
self._cleanupFiles(self._targetPath)
self._copyPythonStaticResources()
if self._verbose:
self._log.write('COMPLETE: Resource Collection')
return True
def _createSetupFile(self, binPath):
path = FileUtils.createPath(binPath, 'setup.py', isFile=True)
scriptPath = inspect.getabsfile(self.applicationClass)
try:
sourcePath = PyGlassEnvironment.getPyGlassResourcePath(
'..', 'setupSource.txt', isFile=True)
f = open(sourcePath, 'r+')
source = f.read()
f.close()
except Exception as err:
print(err)
return None
try:
f = open(path, 'w+')
f.write(source.replace(
'##SCRIPT_PATH##', StringUtils.escapeBackSlashes(scriptPath)
).replace(
'##RESOURCES##', StringUtils.escapeBackSlashes(JSON.asString(self.resources))
).replace(
'##INCLUDES##', StringUtils.escapeBackSlashes(JSON.asString(self.siteLibraries))
).replace(
'##ICON_PATH##', StringUtils.escapeBackSlashes(self._createIcon(binPath))
).replace(
'##APP_NAME##', self.appDisplayName
).replace(
'##SAFE_APP_NAME##', self.appDisplayName.replace(' ', '_') ))
f.close()
except Exception as err:
print(err)
return None
return path
def write(self, session, path, pretty =False, gzipped =True, difference =True):
if self.results is None and not self.process(session, difference):
return False
try:
JSON.toFile(path, self.results, pretty=pretty, gzipped=gzipped, throwError=True)
return True
except Exception as err:
self.logger.writeError([
u'ERROR: Unable to write export file',
u'PATH: ' + StringUtils.toUnicode(path)], err)
return False
def __init__(self, containerPath, isRemoteDeploy =False, sourceRootFolder ='src', **kwargs):
"""Creates a new instance of Site."""
super(Site, self).__init__()
self.errorCount = 0
self.warningCount = 0
self._staticPaths = []
self._logger = ArgsUtils.getLogger(self, kwargs)
self._sourceRootFolderName = sourceRootFolder
# NGinx root path in which all files reside
self._containerPath = FileUtils.cleanupPath(containerPath, isDir=True)
# Location of the source files used to create the website
self._sourceWebRootPath = FileUtils.createPath(containerPath, sourceRootFolder, isDir=True)
# Locations where files should be deployed. If the target root path is None, which is the
# default value, the local web root path is used in its place.
if isRemoteDeploy:
self._targetWebRootPath = FileUtils.cleanupPath(
tempfile.mkdtemp(prefix='staticFlow_'), isDir=True)
else:
self._targetWebRootPath = None
self._localWebRootPath = FileUtils.createPath(
containerPath, ArgsUtils.get('localRootFolder', 'root', kwargs), isDir=True)
path = FileUtils.createPath(self.sourceWebRootPath, '__site__.def', isFile=True)
try:
self._data.data = JSON.fromFile(path, throwError=True)
except Exception, err:
self.writeLogError(u'Unable to load site definition file: "%s"' % path, error=err)
pass
def __init__(self, site):
""" Creates a new instance of RobotFileGenerator """
self.site = site
try:
self._data = JSON.fromFile(self.definitionPath)
except Exception, err:
self._data = [{'USER_AGENT':'*'}]
def _setEnvValue(cls, key, value):
settings = cls._getEnvValue(None) if cls._ENV_SETTINGS is None else cls._ENV_SETTINGS
if settings is None:
settings = dict()
cls._ENV_SETTINGS = settings
if isinstance(key, basestring):
key = [key]
src = settings
for k in key[:-1]:
src = src[k]
src[key[-1]] = value
envPath = cls.getRootLocalResourcePath(cls._GLOBAL_SETTINGS_FILE, isFile=True)
envDir = os.path.dirname(envPath)
if not os.path.exists(envDir):
os.makedirs(envDir)
f = open(envPath, 'w+')
try:
f.write(JSON.asString(cls._ENV_SETTINGS))
except Exception, err:
print 'ERROR: Unable to write environmental settings file at: ' + envPath
return False
def _writeError(self, data):
""" Writes import error data to the logger, formatting it for human readable display. """
source = {}
if 'data' in data:
for n,v in DictUtils.iter(data['data']):
source[' '.join(n.split('_')).title()] = v
indexPrefix = ''
if 'index' in data:
indexPrefix = ' [INDEX: %s]:' % data.get('index', 'Unknown')
result = [
'IMPORT ERROR%s: %s' % (indexPrefix, data['message']),
'DATA: ' + DictUtils.prettyPrint(source)]
if 'existing' in data:
source = {}
snapshot = data['existing'].snapshot
if snapshot:
snapshot = JSON.fromString(snapshot)
if snapshot:
for n,v in DictUtils.iter(snapshot):
source[' '.join(n.split('_')).title()] = v
result.append('CONFLICT: ' + DictUtils.prettyPrint(source))
if 'error' in data:
self._logger.writeError(result, data['error'])
else:
self._logger.write(result)
def _writeImpl(self, value, *args, **kwargs):
if not value:
return u''
elif isinstance(value, dict) or isinstance(value, list):
value = JSON.asString(value)
elif not isinstance(value, basestring):
value = str(value)
value = value.replace("'", "\'").replace('\n',' ')
offset = value.find('\'')
while offset != -1:
if offset == 0 or value[offset-1] != '\\':
value = value[:offset] + '\\' + value[offset:]
offset = value.find('\'', offset + 1)
if not value:
kwargs['writeEmpty'] = False
for j in self._joins:
v = j.write(*args, **kwargs)
if v:
return v
if not ArgsUtils.get('writeEmpty', True, kwargs):
return None
return u'%s=\'%s\'' % (self._name, value)
def add(self, *args, **kwargs):
"""Adds value to the existing item, replacing existing entries.
@@@param value:string
The value argument can be a single value.
@@@param group:string
The name of the group in which to add the value. Default of None adds the value to the
root group.
"""
value = ArgsUtils.get('value', None, kwargs, args, 0)
if value is None:
value = u''
elif isinstance(value, dict) or isinstance(value, list):
value = JSON.asString(value)
else:
value = unicode(value)
group = ArgsUtils.get('group', None, kwargs, args, 1)
once = ArgsUtils.get('once', False, kwargs)
if group:
target = self._tempSubs if once else self._subs
target[group] = value
else:
if once:
self._tempRoot = value
else:
self._root = value
def _parseData(self, data):
if not data:
return None
try:
return JSON.fromString(data)
except Exception, err:
return data
def get(cls, url, config):
"""Returns the embedding object for the specified URL and config."""
try:
req = requests.get(config['url'] + '?url=' + urllib2.quote(url) + '&format=json')
return JSON.fromString(req.text)
except Exception, err:
return None
def flush(self):
if not self._buffer:
return
if sys.platform.startswith('win'):
return
items = []
for b in self._buffer:
try:
d = DictUtils.merge(self._meta, b['data'])
item = b['prefix'] + ' ' + JSON.asString(d)
except Exception as err:
item = '>> EXCEPTION: JSON ENCODING FAILED >> ' + str(err).replace('\n', '\t')
try:
item = item.encode('utf8', 'ignore')
except Exception as err:
item = '>> EXCEPTION: UNICODE DECODING FAILED >> ' + str(err).replace('\n', '\t')
items.append(item)
count = self._fileCount
offset = random.randint(0, count - 1)
success = False
path = self.getReportFolder() + self._timeCode + '/'
if not os.path.exists(path):
os.makedirs(path)
for i in range(count):
index = (i + offset) % count
p = path + str(index) + '.report'
lock = FileLock(p)
if lock.i_am_locking() and i < count - 1:
continue
try:
lock.acquire()
except Exception:
continue
try:
out = StringUtils.toUnicode('\n'.join(items) + '\n')
f = open(p, 'a+')
f.write(out.encode('utf8'))
f.close()
success = True
except Exception as err:
print("REPORTER ERROR: Unable to write report file.")
print(err)
lock.release()
if success:
break
self.clear()
return success
def _saveSettings(self):
if self._settings is None:
return
if not JSON.toFile(self._path, self._settings, pretty=self._pretty):
print('ERROR: Unable to save application settings file: ' + self._path)
return False
return True
def _saveSettings(self):
if self._settings is None:
return
if not JSON.toFile(self._path, self._settings, pretty=self._pretty):
print 'ERROR: Unable to save application settings file: ' + self._path
return False
return True
def _loadBuildSnapshot(self):
settings = SettingsConfig(CompilerDeckEnvironment.projectSettingsPath, pretty=True)
snap = settings.get(['BUILD', 'LAST_SNAPSHOT'])
if snap is None:
return
try:
self._buildSnapshot = JSON.fromString(snap)
except Exception, err:
pass
def handle(self):
try:
data = self.rfile.readline().strip()
self._log.write('HANDLE: ' + str(data))
try:
result = self._respondImpl(JSON.fromString(unquote(data)))
except Exception as err:
self._log.writeError('RESPOND FAILURE', err)
if self.returnResponse:
self.wfile.write(JSON.asString({'error': 1}))
return
if self.returnResponse:
out = {'error': 0}
if result:
out['payload'] = result
self.wfile.write(out)
except Exception as err:
self._log.write('HANDLE FAILURE', err)
return
def _populateTools(self):
"""Doc..."""
path = self.getAppResourcePath('ToolsManifest.json', isFile=True)
try:
f = open(path)
definition = JSON.fromString(f.read())
f.close()
except Exception, err:
self.log.writeError('ERROR: Unable to read tools manifest file.', err)
return
def handle(self):
try:
data = self.rfile.readline().strip()
self._log.write('HANDLE: ' + str(data))
try:
result = self._respondImpl(JSON.fromString(unquote(data)))
except Exception as err:
self._log.writeError('RESPOND FAILURE', err)
if self.returnResponse:
self.wfile.write(JSON.asString({'error':1}))
return
if self.returnResponse:
out = {'error':0}
if result:
out['payload'] = result
self.wfile.write(out)
except Exception as err:
self._log.write('HANDLE FAILURE', err)
return
def _storeBuildSnapshot(self):
if not self._buildSnapshot:
return
snap = dict()
for n,v in self._buildSnapshot.iteritems():
if n in ['parent']:
continue
snap[n] = v
settings = SettingsConfig(CompilerDeckEnvironment.projectSettingsPath, pretty=True)
settings.set(['BUILD', 'LAST_SNAPSHOT'], JSON.asString(snap))
def _loadSettings(self):
"""Doc..."""
if self._settings is not None:
return self._settings
if not os.path.exists(self._path):
self._settings = dict()
return self._settings
self._settings = JSON.fromFile(self._path)
if self._settings is None:
return dict()
return self._settings
def _createErrorResult(self, code =None, info =None, data=None):
out = dict(
success=False,
error=True,
code=code if code else 'COMMUNICATOR_ERROR',
info=info if info else 'Unknown error occurred.',
data=data )
# Keep errors to the 50 most recent to prevent memory overloads on long sessions.
while len(self._errors) > 49:
self._errors.pop(0)
self._errors.append(out)
return JSON.asString(out)
def _loadSettings(self):
"""Doc..."""
if self._settings is not None:
return self._settings
if not os.path.exists(self._path):
self._settings = dict()
return self._settings
self._settings = JSON.fromFile(self._path)
if self._settings is None:
self._settings = dict()
return self._settings
return self._settings
def _processFolderDefinitions(self, path):
cd = ConfigsDict(JSON.fromFile(path))
directory = FileUtils.getDirectoryOf(path)
for item in os.listdir(directory):
# Only find content source file types
if not StringUtils.ends(item, ('.sfml', '.html')):
continue
# Skip files that already have a definitions file
itemPath = FileUtils.createPath(directory, item, isFile=True)
itemDefsPath = itemPath.rsplit('.', 1)[0] + '.def'
if os.path.exists(itemDefsPath):
continue
test = SiteProcessUtils.testFileFilter(
itemPath,
cd.get(('FOLDER', 'EXTENSION_FILTER')),
cd.get(('FOLDER', 'NAME_FILTER')))
if not test:
continue
JSON.toFile(itemDefsPath, dict(), pretty=True)
return True
def _createAttr(self, name, value):
if not value:
return u''
elif isinstance(value, dict) or isinstance(value, list):
value = JSON.asString(value)
elif not isinstance(value, basestring):
value = str(value)
value = value.replace("'", "\'").replace('\n',' ')
offset = value.find('\'')
while offset != -1:
if offset == 0 or value[offset-1] != '\\':
value = value[:offset] + '\\' + value[offset:]
offset = value.find('\'', offset + 1)
return u'%s%s=\'%s\'' % (self._prefix, name, value)
def __init__(self, localRootPath, sourceWebRootPath, forceHtml =False, forceAll =False, **kwargs):
"""Creates a new instance of S3SiteDeployer."""
self._logger = ArgsUtils.getLogger(self, kwargs)
self._localRootPath = FileUtils.cleanupPath(localRootPath, isDir=True)
self._sourceWebRootPath = FileUtils.cleanupPath(sourceWebRootPath, isDir=True)
self._forceHtml = forceHtml
self._forceAll = forceAll
self._cdnRootPath = None
try:
siteData = JSON.fromFile(FileUtils.createPath(
sourceWebRootPath, '__site__.def', isFile=True), throwError=True)
except Exception, err:
self._logger.writeError(
u'Failed to read __site__.def file. Check to make sure JSON is valid.', err)
siteData = {}
def createHeaderFile(cls, path, lastModified =None, headers =None):
if not lastModified:
lastModified = datetime.datetime.utcnow()
if isinstance(lastModified, tuple) or isinstance(lastModified, list):
modTime = lastModified[0]
for newTime in lastModified[1:]:
if newTime and newTime > modTime:
modTime = newTime
lastModified = modTime
if not headers:
headers = dict()
headers['_LAST_MODIFIED'] = TimeUtils.dateTimeToWebTimestamp(lastModified)
return JSON.toFile(path + '.headers', headers)
def _populateTools(self):
"""Doc..."""
path = self.getAppResourcePath('ToolsManifest.json', isFile=True)
try:
f = open(path)
definition = JSON.fromString(f.read())
f.close()
except Exception as err:
self.log.writeError('ERROR: Unable to read tools manifest file.', err)
return
for tool in ArgsUtils.getAsList('tools', definition):
self._addTool(tool)
self._toolBox.layout().addStretch()
def _deployWalker(self, args, path, names):
"""Doc..."""
# Skip CDN file uploads when not walking the CDN root path explicitly
if not args['cdn'] and path.find(StaticFlowEnvironment.CDN_ROOT_PREFIX) != -1:
return
for name in names:
namePath = FileUtils.createPath(path, name)
if os.path.isdir(namePath) or StringUtils.ends(name, self._SKIP_EXTENSIONS):
continue
headersPath = namePath + '.headers'
if os.path.exists(headersPath):
headers = JSON.fromFile(headersPath)
else:
headers = dict()
if self._forceAll:
lastModified = None
elif self._forceHtml and StringUtils.ends(name, self._FORCE_HTML_EXTENSIONS):
lastModified = None
else:
lastModified = ArgsUtils.extract('_LAST_MODIFIED', None, headers)
if lastModified:
lastModified = TimeUtils.webTimestampToDateTime(lastModified)
kwargs = dict(
key=u'/' + namePath[len(self._localRootPath):].replace(u'\\', u'/').strip(u'/'),
maxAge=headers.get('max-age', -1),
eTag=headers.get('eTag', None),
expires=headers.get('Expires'),
newerThanDate=lastModified,
policy=S3Bucket.PUBLIC_READ)
if StringUtils.ends(name, self._STRING_EXTENSIONS):
result = self._bucket.put(
contents=FileUtils.getContents(namePath),
zipContents=True,
**kwargs)
else:
result = self._bucket.putFile(filename=namePath, **kwargs)
if result:
self._logger.write(u'DEPLOYED: ' + unicode(namePath) + u'->' + unicode(kwargs['key']))
def flush(self):
if not self._buffer:
return
if sys.platform.startswith('win'):
return
items = []
for b in self._buffer:
try:
d = dict(self._meta.items() + b['data'].items())
item = b['prefix'] + u' ' + JSON.asString(d)
except Exception, err:
item = '>> EXCEPTION: JSON ENCODING FAILED >> ' + str(err).replace('\n', '\t')
try:
item = item.encode('utf8', 'ignore')
except Exception, err:
item = '>> EXCEPTION: UNICODE DECODING FAILED >> ' + str(err).replace('\n', '\t')
def _getEnvValue(cls, key, defaultValue =None, refresh =False, error =False):
if cls._ENV_SETTINGS is None or refresh:
if not cls.settingsFileExists():
print('WARNING: No environmental settings file found.')
return defaultValue
envPath = cls.getRootLocalResourcePath(cls._GLOBAL_SETTINGS_FILE, isFile=True)
f = open(envPath, 'r+')
try:
res = f.read()
except Exception:
print('ERROR: Unable to read the environmental settings file at: ' + envPath)
return
finally:
f.close()
try:
settings = JSON.fromString(res)
except Exception:
print('ERROR: Unable to parse environmental settings file at: ' + envPath)
return
cls._ENV_SETTINGS = settings
else:
settings = cls._ENV_SETTINGS
if key is None:
return settings
if StringUtils.isStringType(key):
key = [key]
value = settings
for k in key:
if k in value:
value = value[k]
else:
if error:
raise Exception('Missing environmental setting: ' + str(key))
return defaultValue
return value
def __init__(self, request, rootPackage =None, **kwargs):
""" Creates a new ApiRouterView instance.
@param rootPackage - The root package in which the router will import views. By default
the module will look in same package as the router class. Packages can be absolute,
or relative to the current package. """
super(ApiRouterView, self).__init__(request, **kwargs)
# Determine root package
self._root = rootPackage if rootPackage else ClassUtils.getModulePackage(self.__class__, 1)
zargs = self.getArg('zargs', None)
if zargs:
try:
self._zargs = JSON.fromString(zargs)
except Exception as err:
self._zargs = None
else:
self._zargs = None
self._signature = StringUtils.toUnicode(self.getArg('sig', ''))
self._incomingTimestamp = None
self._outgoingTimestamp = None
def activate(self):
"""Doc..."""
self.widget.aneList.clear()
path = CompilerDeckEnvironment.getProjectPath('NativeExtensions', isDir=True)
for item in os.listdir(path):
itemPath = FileUtils.createPath(path, item)
if not os.path.isdir(itemPath):
continue
settingsPath = FileUtils.createPath(itemPath, 'compiler', 'settings.vcd', isFile=True)
if not os.path.exists(settingsPath):
continue
settings = JSON.fromFile(settingsPath)
if not settings:
continue
DataListWidgetItem(settings.get('LABEL', item), self.widget.aneList, ident=item, data={
'folder':item,
'path':itemPath,
'settings':settings })
self.widget.aneList.sortItems()
def sha256hmacSign(cls, key, **kwargs):
return cls.sha256hmac(key, JSON.asString(kwargs))
def sha256hmacSignObject(cls, key, obj):
return cls.sha256hmac(key, JSON.asString(obj))