def _handleResponseReady(self, request, response):
"""Event handler for the response object being ready for use."""
if self._cacheControlPublic:
response.cache_control = "public"
# -------------------------------------------------------------------------------------------
# Cache Expiration: Set the caching values according to the _expires property
rep = self._explicitResponse
if rep is None or (isinstance(rep, ViewResponse) and rep.allowCaching):
response.cache_control.max_age = self.expires if not self.expires is None else 0
else:
response.cache_control.max_age = 0
# -------------------------------------------------------------------------------------------
# Cache Validators
if self._etag is not None:
response.etag = StringUtils.toUnicode(self._etag)
if self._lastModified is not None:
response.last_modified = self._lastModified
# If required encode the response headers as strings to prevent unicode errors. This is
# necessary for certain WSGI server applications, e.g. flup.
if self.ziggurat.strEncodeEnviron:
for n, v in DictUtils.iter(response.headers):
if StringUtils.isStringType(v):
response.headers[n] = StringUtils.toStr2(v)
# Clean up per-thread sessions.
ConcreteModelsMeta.cleanupSessions()
def _importCompiled(cls, widgetPath):
"""_importCompiled doc..."""
if imp:
return imp.load_compiled('PySideUiFileSetup', StringUtils.toStr2(widgetPath))
loader = importlib.machinery.SourcelessFileLoader('PySideUiFileSetup', widgetPath)
return loader.load_module()
def _toggleSideWidget(self, open =True):
"""_openWidget doc..."""
if self._anim:
self._anim.stop()
self._anim.deleteLater()
self._anim = None
self.sideWidget.show()
anim = QtCore.QPropertyAnimation(self.sideWidget, StringUtils.toStr2('pos'))
anim.setDuration(250)
if self.dockOnLeft:
outValue = -self._getSideWidgetWidth()
inValue = 0
else:
w = self.size().width()
outValue = w
inValue = w - self._getSideWidgetWidth()
anim.setStartValue(QtCore.QPoint(outValue if open else inValue, 0))
anim.setEndValue(QtCore.QPoint(inValue if open else outValue, 0))
anim.finished.connect(self._handleAnimFinished)
self._anim = anim
anim.start()
def putContents(cls,
content,
path,
append=False,
raiseErrors=False,
gzipped=False):
if not StringUtils.isStringType(content):
content = ''
content = StringUtils.toStr2(content)
try:
mode = 'a+' if append and os.path.exists(path) else 'w+'
if gzipped:
f = open(path, mode)
else:
f = open(path, mode)
f.write(content)
f.close()
except Exception as err:
if raiseErrors:
raise
print(err)
return False
return True
def loads(self, srcString, srcType):
if srcString is None:
self._log.write('ERROR: Source string is empty or invalid.')
return False
srcString = StringUtils.toStr2(srcString)
self._path = None
self._src = srcString
self._type = srcType
return True
def flush(self, **kwargs):
if not self._buffer:
return
items = []
for logItem in self._buffer:
item = self.logMessageToString(logMessage=logItem) + '\n'
item = StringUtils.toStr2(item)
items.append(item)
for cb in self._writeCallbacks:
try:
cb(self, items)
except Exception:
pass
if not self._logPath:
self.clear()
return
elif self._logFile:
try:
out = StringUtils.toStr2('\n').join(items)
exists = os.path.exists(self._logFile)
with FileLock(self._logFile, 'a') as lock:
lock.file.write(out)
lock.release()
try:
if not exists and not OsUtils.isWindows():
os.system('chmod 775 %s' % self._logFile)
except Exception:
pass
except Exception as err:
print("LOGGER ERROR: Unable to write log file.")
print(err)
self.clear()
def cleanDictKeys(cls, source, force =False):
""" Python 2.6 and below don't allow unicode argument keys, so these must be converted to
byte strings explicitly to prevent exceptions.
"""
vi = sys.version_info
if not force and (vi[0] >= 3 or (vi[1] >= 7 and vi[2] >= 5)):
return source
out = dict()
for n,v in cls.iter(source):
out[StringUtils.toStr2(n, True)] = v
return out
def flush(self, **kwargs):
if not self._buffer:
return
items = []
for logItem in self._buffer:
item = self.logMessageToString(logMessage=logItem) + '\n'
item = StringUtils.toStr2(item)
items.append(item)
for cb in self._writeCallbacks:
try:
cb(self, items)
except Exception:
pass
if not self._logPath:
self.clear()
return
elif self._logFile:
try:
out = StringUtils.toStr2('\n').join(items)
exists = os.path.exists(self._logFile)
with FileLock(self._logFile, 'a') as lock:
lock.file.write(out)
lock.release()
try:
if not exists and not OsUtils.isWindows():
os.system('chmod 775 %s' % self._logFile)
except Exception:
pass
except Exception as err:
print("LOGGER ERROR: Unable to write log file.")
print(err)
self.clear()
def _postAnalyze(self):
self.logger.write('%s gauge calculated tracks' % self._count)
self._trackwayCsv.save()
csv = CsvWriter(
path=self.getPath('Simple-Gauge-Errors.csv', isFile=True),
autoIndexFieldName='Index',
fields=[
('uid', 'UID'),
('fingerprint', 'Fingerprint') ])
for track in self._errorTracks:
csv.createRow(uid=track.uid, fingerprint=track.fingerprint)
csv.save()
if self._errorTracks:
self.logger.write('Failed to calculate gauge for %s tracks' % len(self._errorTracks))
csv = CsvWriter(
path=self.getPath('Simple-Gauge-Ignores.csv', isFile=True),
autoIndexFieldName='Index',
fields=[
('uid', 'UID'),
('fingerprint', 'Fingerprint') ])
for track in self._ignoreTracks:
csv.createRow(uid=track.uid, fingerprint=track.fingerprint)
csv.save()
if self._ignoreTracks:
self.logger.write('%s tracks lacked suitability for gauge calculation' % len(
self._ignoreTracks))
plotData = [
('stride', 'red', 'AU', 'Stride-Normalized Weighted'),
('pace', 'green', 'AU', 'Pace-Normalized Weighted'),
('width', 'blue', 'AU', 'Width-Normalized Weighted'),
('abs', 'purple', 'm', 'Absolute Unweighted') ]
for data in plotData:
out = []
source = ListUtils.sortListByIndex(
source=getattr(self._trackwayGauges, StringUtils.toStr2(data[0])),
index=0,
inPlace=True)
for item in source:
out.append(PositionValue2D(x=len(out), y=item[1].value, yUnc=item[1].uncertainty))
self._plotTrackwayGauges(out, *data[1:])
self.mergePdfs(self._paths, 'Gauges.pdf')
def toFile(cls, path, value, pretty =False, gzipped =False, throwError =False):
try:
res = StringUtils.toStr2(cls.asString(value, pretty=pretty))
if gzipped:
f = gzip.open(path, 'wb')
f.write(StringUtils.toBytes(res))
else:
f = open(path, 'w+')
f.write(res)
f.close()
return True
except Exception as err:
if throwError:
raise
else:
print(err)
return False
def createRevision(
cls, databaseUrl, message, resourcesPath =None, localResourcesPath =None, info =None
):
config = cls.getConfig(
databaseUrl=databaseUrl,
resourcesPath=resourcesPath,
localResourcesPath=localResourcesPath)
previousRevisions = cls.getRevisionList(
databaseUrl=databaseUrl,
resourcesPath=resourcesPath,
config=config)
alembicCmd.revision(
config=config,
message=StringUtils.toUnicode(len(previousRevisions)) + ': ' + message)
if not info:
return True
scriptInfo = alembicScript.ScriptDirectory.from_config(config)
scriptPath = None
for item in os.listdir(scriptInfo.versions):
if item.startswith(scriptInfo.get_current_head()):
scriptPath = os.path.join(scriptInfo.versions, item)
break
if not scriptPath:
return True
info = StringUtils.toUnicode(info)
f = open(scriptPath, 'r+')
script = StringUtils.toUnicode(f.read())
f.close()
index = script.find('"""')
index = script.find('"""', index + 1)
script = script[:index] + info + '\n' + script[index:]
f = open(scriptPath, 'w+')
f.write(StringUtils.toStr2(script))
f.close()
return True
def mergePdfs(self, paths, fileName =None):
""" Takes a list of paths to existing PDF files and merges them into a single pdf with
the given file name.
[fileName] :: String :: None
The name of the file to be written. If not specified, a file name will be created
using the name of this class. """
merger = PdfFileMerger()
for p in paths:
with open(p, 'rb') as f:
merger.append(PdfFileReader(f))
if not fileName:
fileName = '%s-Report.pdf' % self.__class__.__name__
if not StringUtils.toStr2(fileName).endswith('.pdf'):
fileName += '.pdf'
with open(self.getPath(fileName, isFile=True), 'wb') as f:
merger.write(f)
def toFile(cls,
path,
value,
pretty=False,
gzipped=False,
throwError=False):
try:
res = StringUtils.toStr2(cls.asString(value, pretty=pretty))
if gzipped:
f = gzip.open(path, 'wb')
f.write(StringUtils.toBytes(res))
else:
f = open(path, 'w+')
f.write(res)
f.close()
return True
except Exception as err:
if throwError:
raise
else:
print(err)
return False
def __str__(self):
modelInfo = self._getPrintAttrs()
if isinstance(modelInfo, dict):
out = ""
for n, v in DictUtils.iter(modelInfo):
out += " " + StringUtils.toUnicode(n) + "[" + StringUtils.toUnicode(v) + "]"
modelInfo = out
elif modelInfo:
modelInfo = " " + StringUtils.toUnicode(modelInfo)
else:
modelInfo = ""
return StringUtils.toStr2(
"<%s[%s] cts[%s] upts[%s]%s>"
% (
self.__class__.__name__,
StringUtils.toUnicode(self.i),
StringUtils.toUnicode(self.cts.strftime("%m-%d-%y %H:%M:%S") if self.cts else "None"),
StringUtils.toUnicode(self.upts.strftime("%m-%d-%y %H:%M:%S") if self.upts else "None"),
modelInfo,
)
)
def putContents(
cls, content, path, append =False, raiseErrors =False,
gzipped =False
):
if not StringUtils.isStringType(content):
content = ''
content = StringUtils.toStr2(content)
try:
mode = 'a+' if append and os.path.exists(path) else 'w+'
if gzipped:
f = open(path, mode)
else:
f = open(path, mode)
f.write(content)
f.close()
except Exception as err:
if raiseErrors:
raise
print(err)
return False
return True
def __new__(mcs, name, bases, attrs):
name = StringUtils.toStr2(name)
for n, v in list(attrs.items()):
attrName = n[1:]
if isinstance(v, Column) and n.startswith('_') and not attrName in attrs:
v.key = attrName
v.name = attrName
# Add dynamic property
attrs[attrName] = hybrid_property(
ModelPropertyGetter(n),
ModelPropertySetter(n),
None,
ModelPropertyExpression(n))
# Add external-key property
info = getattr(v, 'info')
if info and 'model' in info:
columnName = info['column'] if 'column' in info else 'i'
attrs[info['get']] = property(
ExternalKeyProperty(attrName, info['model'], columnName))
return DeclarativeMeta.__new__(mcs, name, bases, attrs)
def modelsInit(cls, databaseUrl, initPath, initName):
out = dict()
zipIndex = initPath[0].find(cls._ZIP_FIND)
if zipIndex == -1:
moduleList = os.listdir(initPath[0])
else:
splitIndex = zipIndex + len(cls._ZIP_FIND)
zipPath = initPath[0][: splitIndex - 1]
modulePath = initPath[0][splitIndex:]
z = zipfile.ZipFile(zipPath)
moduleList = []
for item in z.namelist():
item = os.sep.join(item.split("/"))
if item.startswith(modulePath):
moduleList.append(item.rsplit(os.sep, 1)[-1])
# Warn if module initialization occurs before pyglass environment initialization and
# then attempt to initialize the environment automatically to prevent errors
if not PyGlassEnvironment.isInitialized:
cls.logger.write(
StringUtils.dedent(
"""
[WARNING]: Database initialization called before PyGlassEnvironment initialization.
Attempting automatic initialization to prevent errors."""
)
)
PyGlassEnvironment.initializeFromInternalPath(initPath[0])
if not cls.upgradeDatabase(databaseUrl):
cls.logger.write(
StringUtils.dedent(
"""
[WARNING]: No alembic support detected. Migration support disabled."""
)
)
items = []
for module in moduleList:
if module.startswith("__init__.py") or module.find("_") == -1:
continue
parts = module.rsplit(".", 1)
parts[0] = parts[0].rsplit(os.sep, 1)[-1]
if not parts[-1].startswith(StringUtils.toStr2("py")) or parts[0] in items:
continue
items.append(parts[0])
m = None
n = None
r = None
c = None
try:
n = module.rsplit(".", 1)[0]
m = initName + "." + n
r = __import__(m, locals(), globals(), [n])
c = getattr(r, StringUtils.toText(n))
out[n] = c
if not c.__table__.exists(c.ENGINE):
c.__table__.create(c.ENGINE, True)
except Exception as err:
cls._logger.writeError(
[
"MODEL INITIALIZATION FAILURE:",
"INIT PATH: %s" % initPath,
"INIT NAME: %s" % initName,
"MODULE IMPORT: %s" % m,
"IMPORT CLASS: %s" % n,
"IMPORT RESULT: %s" % r,
"CLASS RESULT: %s" % c,
],
err,
)
return out
def runPythonImport(cls, payload):
try:
kwargs = payload.get('kwargs', {})
targetModule = StringUtils.toStr2(payload.get('module'))
targetMethod = StringUtils.toStr2(payload.get('method'))
targetClass = StringUtils.toStr2(payload.get('class'))
target = targetClass if targetClass is not None else targetMethod
if target is None:
parts = targetModule.rsplit('.', 1)
targetModule = parts[0]
target = parts[1]
except Exception as err:
NimbleEnvironment.logError([
'ERROR: Failed to parse python import payload',
'PAYLOAD: ' + DictUtils.prettyPrint(payload)], err)
return NimbleResponseData(
kind=DataKindEnum.PYTHON_IMPORT,
error=cls._getDetailedError('\n'.join([
'ERROR: Failed to parse python import payload',
'PAYLOAD: ' + DictUtils.prettyPrint(payload)]), err),
response=NimbleResponseData.FAILED_RESPONSE)
# Dynamically import the specified module and reload it to make sure any changes have
# been updated
try:
module = __import__(
StringUtils.toStr2(targetModule),
globals(), locals(),
[StringUtils.toStr2(target)] if target else [])
reload(module)
target = getattr(module, target)
except Exception as err:
NimbleEnvironment.logError([
'ERROR: Failed to import python target',
'MODULE: %s' % targetModule,
'TARGET: %s' % target,
'PAYLOAD: ' + DictUtils.prettyPrint(payload)], err)
return NimbleResponseData(
kind=DataKindEnum.PYTHON_IMPORT,
error=cls._getDetailedError(
'Failed to import python module', err),
response=NimbleResponseData.FAILED_RESPONSE)
try:
result = dict()
if targetClass is not None:
tc = target()
result = getattr(tc, targetMethod)(**kwargs) \
if targetMethod else \
tc(**kwargs)
elif targetMethod is not None:
result = target(**kwargs)
else:
# Find a NimbleScriptBase derived class definition and if it exists, run it to
# populate the results
for name,value in DictUtils.iter(Reflection.getReflectionDict(target)):
if not inspect.isclass(value):
continue
if NimbleScriptBase in value.__bases__:
result = getattr(target, name)()(**kwargs)
found = True
# If a result dictionary contains an error key format the response as a failure
errorMessage = None
try:
errorMessage = ArgsUtils.extract(
NimbleEnvironment.REMOTE_RESULT_ERROR_KEY, None, result)
except Exception as err:
pass
return cls.createReply(DataKindEnum.PYTHON_IMPORT, result, errorMessage=errorMessage)
except Exception as err:
msg = 'ERROR: Failed to execute remote script'
NimbleEnvironment.logError([
msg,
'PAYLOAD: ' + DictUtils.prettyPrint(payload),
'TARGET: ' + str(target)], err)
return NimbleResponseData(
kind=DataKindEnum.PYTHON_IMPORT,
error=cls._getDetailedError(msg, err),
response=NimbleResponseData.FAILED_RESPONSE)
def __unicode__(self):
return StringUtils.toStr2(self.__str__())
def runPythonImport(cls, payload):
try:
kwargs = payload.get('kwargs', {})
targetModule = StringUtils.toStr2(payload.get('module'))
targetMethod = StringUtils.toStr2(payload.get('method'))
targetClass = StringUtils.toStr2(payload.get('class'))
target = targetClass if targetClass is not None else targetMethod
if target is None:
parts = targetModule.rsplit('.', 1)
targetModule = parts[0]
target = parts[1]
except Exception as err:
NimbleEnvironment.logError([
'ERROR: Failed to parse python import payload',
'PAYLOAD: ' + DictUtils.prettyPrint(payload)
], err)
return NimbleResponseData(
kind=DataKindEnum.PYTHON_IMPORT,
error=cls._getDetailedError(
'\n'.join([
'ERROR: Failed to parse python import payload',
'PAYLOAD: ' + DictUtils.prettyPrint(payload)
]), err),
response=NimbleResponseData.FAILED_RESPONSE)
# Dynamically import the specified module and reload it to make sure any changes have
# been updated
try:
module = __import__(StringUtils.toStr2(targetModule), globals(),
locals(),
[StringUtils.toStr2(target)] if target else [])
reload(module)
target = getattr(module, target)
except Exception as err:
NimbleEnvironment.logError([
'ERROR: Failed to import python target',
'MODULE: %s' % targetModule,
'TARGET: %s' % target,
'PAYLOAD: ' + DictUtils.prettyPrint(payload)
], err)
return NimbleResponseData(
kind=DataKindEnum.PYTHON_IMPORT,
error=cls._getDetailedError('Failed to import python module',
err),
response=NimbleResponseData.FAILED_RESPONSE)
try:
result = dict()
if targetClass is not None:
tc = target()
result = getattr(tc, targetMethod)(**kwargs) \
if targetMethod else \
tc(**kwargs)
elif targetMethod is not None:
result = target(**kwargs)
else:
# Find a NimbleScriptBase derived class definition and if it exists, run it to
# populate the results
for name, value in DictUtils.iter(
Reflection.getReflectionDict(target)):
if not inspect.isclass(value):
continue
if NimbleScriptBase in value.__bases__:
result = getattr(target, name)()(**kwargs)
found = True
# If a result dictionary contains an error key format the response as a failure
errorMessage = None
try:
errorMessage = ArgsUtils.extract(
NimbleEnvironment.REMOTE_RESULT_ERROR_KEY, None, result)
except Exception as err:
pass
return cls.createReply(DataKindEnum.PYTHON_IMPORT,
result,
errorMessage=errorMessage)
except Exception as err:
msg = 'ERROR: Failed to execute remote script'
NimbleEnvironment.logError([
msg, 'PAYLOAD: ' + DictUtils.prettyPrint(payload),
'TARGET: ' + str(target)
], err)
return NimbleResponseData(
kind=DataKindEnum.PYTHON_IMPORT,
error=cls._getDetailedError(msg, err),
response=NimbleResponseData.FAILED_RESPONSE)
def _runImpl(self):
model = Tracks_SiteMap.MASTER
session = self._session if self._session else model.createSession()
try:
self._log.write(u'<h1>Beginning Sitemap Import...</h1>')
if self._path is None or not os.path.exists(self._path):
self._log.write(u'<h2>Invalid or missing path</h2>')
return 1
# Delete all existing rows
rowCount = session.query(model).count()
if rowCount > 0:
session.query(model).delete()
with open(self._path, 'rU') as f:
try:
reader = csv.reader(
f,
delimiter=StringUtils.toStr2(','),
quotechar=StringUtils.toStr2('"'))
except Exception as err:
self._writeError({
'message':u'ERROR: Unable to read CSV file "%s"' % self._path,
'error':err })
return
if reader is None:
self._writeError({
'message':u'ERROR: Failed to create CSV reader for file "%s"' % self._path })
return
for row in reader:
# Skip any rows that don't start with the proper numeric index value, which
# includes the header row (if it exists) with the column names
try:
index = int(row[0])
except Exception as err:
continue
rowDict = dict()
for column in Reflection.getReflectionList(SitemapCsvColumnEnum):
value = row[column.index]
value = StringUtils.strToUnicode(value)
if value != u'' or value is None:
rowDict[column.name] = value
self._fromSpreadsheetEntry(rowDict, session)
except Exception as err:
if not self._session:
session.rollback()
session.close()
self._log.writeError(u'ERROR: Sitemap Importing Error', err)
return 1
if self._session is None:
session.commit()
session.close()
self._log.write(u'<h1>Sitemap Import Complete</h1>')
return 0
def __str__(self):
"""__str__ doc..."""
return StringUtils.toStr2(self.__unicode__())
def __str__(self):
return StringUtils.toStr2('<%s[%s] uid[%s] %s>' % (
self.__class__.__name__,
StringUtils.toUnicode(self.i),
StringUtils.toUnicode(self.uid),
StringUtils.toUnicode(self.fingerprint)))
def __str__(self):
"""__str__ doc..."""
return StringUtils.toStr2(
'<Trackway[%s] %s "%s">' % (self.i, self.index, self.name))
def __new__(mcs, name, bases, attrs):
binding = ConcretePyGlassModelsMeta._engines.get(name)
if binding is None:
# Retrieve the database URL from the __init__.py for the package in which the model class
# resides.
try:
module = attrs['__module__']
package = module[:module.rfind('.')]
res = __import__(package, globals(), locals(), [StringUtils.toStr2('DATABASE_URL')])
except Exception as err:
PyGlassModelUtils.logger.writeError([
'ERROR: Unable to initialize model',
'NAME: %s' % name ])
raise
try:
sourceUrl = getattr(res, 'DATABASE_URL')
except Exception as err:
PyGlassModelUtils.logger.writeError([
'ERROR: Unable to get DATABASE_URL from %s.__init__.py' % package,
'NAME: %s' % name ], err)
raise
try:
url = PyGlassModelUtils.getEngineUrl(sourceUrl)
except Exception as err:
PyGlassModelUtils.logger.writeError([
'ERROR: Unable to get url from database url',
'DATABASE URL: ' + StringUtils.toUnicode(sourceUrl) ], err)
raise
try:
engine = create_engine(url, echo=False)
except Exception as err:
PyGlassModelUtils.logger.writeError([
'DATABASE FAILURE: Unable to create engine for database',
'URL: ' + StringUtils.toUnicode(url),
'NAME: ' + StringUtils.toUnicode(name)
], err)
raise
try:
Session = scoped_session(sessionmaker(bind=engine))
except Exception as err:
PyGlassModelUtils.logger.writeError([
'DATABASE FAILURE: Unable to create bound Session.',
'URL: ' + StringUtils.toUnicode(url),
'ENGINE: ' + StringUtils.toUnicode(engine)
], err)
raise
try:
Base = declarative_base(bind=engine)
except Exception as err:
PyGlassModelUtils.logger.writeError([
'DATABASE FAILURE: Unable to create database engine Base.',
'URL: ' + StringUtils.toUnicode(url),
'ENGINE: ' + StringUtils.toUnicode(engine)
], err)
raise
try:
Base.metadata.create_all(engine)
except Exception as err:
PyGlassModelUtils.logger.writeError([
'DATABASE FAILURE: Unable to create models.'
'URL: ' + StringUtils.toUnicode(url),
'ENGINE: ' + StringUtils.toUnicode(engine),
'BASE: ' + StringUtils.toUnicode(Base)
], err)
raise
binding = {
'engine':engine,
'SessionClass':Session,
'BaseClass':Base,
'url':url,
'databaseUrl':sourceUrl }
mcs._engines[name] = binding
attrs['ENGINE'] = binding['engine']
attrs['SESSION_CLASS'] = binding['SessionClass']
attrs['BASE'] = binding['BaseClass']
attrs['URL'] = binding['url']
attrs['MODEL_NAME'] = name
attrs['DATABASE_URL'] = binding['databaseUrl']
attrs['BINDING'] = binding
# Add the declarative base to inheritance
declaredBase = (binding['BaseClass'],)
try:
bases = bases + declaredBase
except Exception as err:
bases = declaredBase
out = AbstractPyGlassModelsMeta.__new__(mcs, name, bases, attrs)
return out
def __unicode__(self):
return StringUtils.toStr2(self.__str__())
def __str__(self):
"""__str__ doc..."""
return StringUtils.toStr2(self.__unicode__())
