def loadHtml(url):
try:
DEBUG('htmlLoader: request start', {
'url': url
})
response = urllib2.urlopen(url)
responseDict = utils.dictFromClass(response)
DEBUG('htmlLoader: request successful', {
'url': url,
'status': '{code} ({msg})'.format(**responseDict),
'content-length': int(response.headers.dict['content-length']),
'type': response.headers.type,
})
data = response.read()
DEBUG('htmlLoader: data readed', {
'url': url,
# 'data': utils.BlockString(data, 500),
'data': utils.truncateLongString(data, 600),
})
return data
except Exception, error:
DEBUG('htmlLoader: error catched', {
'url': url,
'error': errors.toBlockString(error),
})
def sendImageFile(id):
# return render_template('viewImage.html', id=id)
uploadPath = config['uploadPath']
if id == 'last' or id == 'recent':
id = imageUtils.getLastImageId()
imageFilePath = path.join(uploadPath, id + config['imageExt'])
yamlFilePath = path.join(uploadPath, id + '.yaml')
DEBUG('sendImageFile: started', {
'id': id,
'imageFilePath': imageFilePath,
'yamlFilePath': yamlFilePath,
})
if not path.isfile(imageFilePath) or not path.isfile(yamlFilePath):
# return 'imageNotFound'
return render_template('imageNotFound.html', id=id)
with open(yamlFilePath) as yamlFilePathFile:
params = yaml.load(yamlFilePathFile, Loader=yaml.FullLoader)
mimeType = params['mimeType']
fileSize = path.getsize(imageFilePath)
DEBUG(
'sendImageFile: image found', {
'imageFilePath': imageFilePath,
'yamlFilePath': yamlFilePath,
'fileSize': int(fileSize),
'params': params,
})
# params data sample:
# - ext: jpg
# - filename: test-image.jpg
# - id: 200930-225108
# - mimeType: image/jpeg
# - name: test-image
# - timestamp: 2020.09.30-22:51:08
return send_file(imageFilePath, mimetype=mimeType)
def removeAllImages():
"""
Remove all images
"""
uploadPath = config['uploadPath']
# filelist = [ file for file in os.listdir(mydir) if file.endswith('.bak') ] # TODO: Filter example
DEBUG('imageUtils:removeAllImages', {'uploadPath': uploadPath})
for file in os.listdir(uploadPath):
DEBUG('imageUtils:removeAllImages: remove file', {'file': file})
os.remove(os.path.join(uploadPath, file))
def sign_up():
data = request.json
username = data['username']
password = data['password']
DEBUG("username is %s, password is %s" % (username, password))
DEBUG("users is %s" % user_manager.users)
if username not in user_manager.users:
user_manager.save_user(User(username, password))
return success(message="User %s sign up success" % username)
return failed(rtn=ErrorCode.USER_ALREADY_EXISTS, message="User %s already exists" % (username))
def getImageData(id):
list = imageUtils.loadImagesList(True)
DEBUG('externalApi:getImageData called (/api/images/' + id + ')', {
'id': id,
'list': list,
})
for index, data in enumerate(list):
if data['id'] == id:
DEBUG('externalApi:getImageData called (/api/images/' + id + ')', {
'id': id,
'list': list,
})
return jsonify(data)
return jsonify({'error': 'No image found for id ' + id})
def uploadImage(ip, file):
filename = file.filename
name, extension = path.splitext(filename)
ext = extension[1:].lower()
now = datetime.datetime.now()
timestamp = now.strftime(config['shortDateFormat'])
# timestamp = now.strftime(config['preciseDateFormat'])
id = ip + '-' + now.strftime(config['dateTagPreciseFormat'])
data = {
'ip': ip,
'filename': filename,
'name': name,
'ext': ext,
'timestamp': timestamp,
}
if ext not in mimeExtensions:
error = 'Unexpected extension (' + ext + ')'
DEBUG('uploadImage: error: ' + error, data)
return {'error': error}
data['mimeType'] = mimeTypes[ext]
DEBUG('uploadImage: data', data)
uploadPath = config['uploadPath']
try:
if not os.path.exists(uploadPath):
os.makedirs(uploadPath)
# Save image...
imageFilePath = os.path.join(uploadPath, id + config['imageExt'])
file.save(imageFilePath)
# Save yaml...
yamlFilePath = os.path.join(uploadPath, id + '.yaml')
yaml.safe_dump(data,
open(yamlFilePath, 'wb'),
encoding='utf-8',
allow_unicode=True)
# Update index file...
indexFilePath = os.path.join(uploadPath, config['imagesIndex'])
with open(indexFilePath, 'ab') as indexFile:
indexFile.write(id + '\t' + ip + '\t' + timestamp + '\n')
except Exception, error:
DEBUG('uploadImage: error catched', {
'error': errors.toBlockString(error),
})
return {'error': 'Upload file error (see server log)'}
def viewImage(id):
# return render_template('viewImage.html', id=id)
uploadPath = config['uploadPath']
yamlFilePath = path.join(uploadPath, id + '.yaml')
if not path.isfile(yamlFilePath):
# return 'imageNotFound'
return render_template('imageNotFound.html', id=id)
with open(yamlFilePath) as yamlFilePathFile:
params = yaml.load(yamlFilePathFile, Loader=yaml.FullLoader)
timestamp = params['timestamp']
ip = params['ip']
imageWidth = config['imageWidth']
imageHeight = config['imageHeight']
DEBUG('viewImage called', {
'yamlFilePath': yamlFilePath,
'params': params,
})
# params data sample:
# - ext: jpg
# - filename: test-image.jpg
# - id: 200930-225108
# - mimeType: image/jpeg
# - name: test-image
# - timestamp: 2020.09.30-22:51:08
return render_template('viewImage.html',
id=id,
ip=ip,
timestamp=timestamp,
imageWidth=imageWidth,
imageHeight=imageHeight,
params=params)
def deleteAllImages():
"""
Remove all images
"""
DEBUG('externalApi:deleteAllImages called (DELETE /api/images)')
imageUtils.removeAllImages()
return jsonify({'success': True})
def listAllImages():
list = imageUtils.loadImagesList(True)
DEBUG('listAllImages called', {
'list': list,
})
if not list or not len(list):
return render_template('noImages.html')
return render_template('listAllImages.html', list=list)
def fallback(route):
debugData = {
'route': route,
'method': request.method,
}
DEBUG('externalApi:fallback: Method is not defined', debugData)
errorStr = 'The route \'/api/{route}\' is not implemented for method \'{method}\''.format(
**debugData)
return jsonify({'error': errorStr})
def removeImages(ids):
"""
Remove specified images
NOTE: Not fails if some of items or item files not exists
"""
# TODO: Lock file?
uploadPath = config['uploadPath']
# Load index..
imagesList = loadImagesList(True)
# filelist = [ file for file in os.listdir(mydir) if file.endswith('.bak') ] # TODO: Filter example
DEBUG('imageUtils:removeImages: start', {
'ids': ids,
'imagesList': imagesList,
'uploadPath': uploadPath,
})
# Remove items from images list...
imagesList = list(filter(lambda item: item['id'] not in ids, imagesList))
# Save updated index file...
indexFilePath = os.path.join(uploadPath, config['imagesIndex'])
DEBUG('imageUtils:removeImages: updated list', {
'ids': ids,
'imagesList': imagesList,
'indexFilePath': indexFilePath,
})
with open(indexFilePath, 'wb') as indexFile:
for item in imagesList:
indexFile.write(item['id'] + '\t' + item['ip'] + '\t' +
item['timestamp'] + '\n')
# Remove single files...
for id in ids:
imageFilePath = os.path.join(uploadPath, id + config['imageExt'])
yamlFilePath = os.path.join(uploadPath, id + '.yaml')
DEBUG(
'imageUtils:removeImages: remove item files', {
'id': id,
'imageFilePath': imageFilePath,
'yamlFilePath': yamlFilePath,
})
# Try to dlete files ignoring 'not exist' errors...
try:
os.remove(imageFilePath)
DEBUG('imageUtils:removeImages: deleted',
{'imageFilePath': imageFilePath})
except Exception, e:
if e.errno == errno.ENOENT:
DEBUG('imageUtils:removeImages: file not exist',
{'imageFilePath': imageFilePath})
else:
raise
try:
os.remove(yamlFilePath)
DEBUG('imageUtils:removeImages: deleted',
{'yamlFilePath': yamlFilePath})
except Exception, e:
if e.errno == errno.ENOENT:
DEBUG('imageUtils:removeImages: file not exist',
{'yamlFilePath': yamlFilePath})
else:
raise
def deleteImages(ids):
"""
Remove specified images
NOTE: Not fails if some of items or item files not exists
"""
DEBUG('externalApi:deleteImages called (DELETE /api/images)', {
'ids': ids,
})
imageUtils.removeImages(ids)
return jsonify({'success': True, 'ids': ids})
def listAllImages():
images = imageUtils.loadImagesList(True)
DEBUG('externalApi:listAllImages called (/api/images)', {
'images': images,
})
if not images:
images = []
# return jsonify(images) # Invalid response: JSONArray instead JSONObject
# return jsonify({'error': 'Test error'}) # Test failback
return jsonify({'images': images})
def addNewUrls(urls):
global testing
rootPath = config['rootPath']
collectorFileName = config[
'collectorFileName'] if not testing else 'collector-test.txt'
collectorFilePath = path.join(rootPath, collectorFileName)
if testing:
DEBUG('collector: testing for addNewUrls', {
'testing': testing,
'collectorFilePath': collectorFilePath,
})
return addNewUrlsToFile(collectorFilePath, urls)
def get_room(room_id):
try:
room = room_manager.get_room(room_id)
DEBUG(room.players)
return success(data={
"players": [{
"name": player.name,
"hero": cards_name_map[player.starting_hero],
"deck": [cards_name_map[card] for card in player.starting_deck]
} for player in room.players]
})
except RoomNotExists:
return failed(rtn=ErrorCode.ROOM_NOT_EXISTS, message="Room %s does not exist" % room_id)
def handle_error(e):
# errorType, errorValue, errorTraceback = sys.exc_info()
# @see https://docs.python.org/2/library/traceback.html
errorTraceback = traceback.format_exc()
error = str(e)
errorRepr = e.__repr__()
errorData = {
'error': error,
'errorRepr': errorRepr,
# 'type': str(errorType),
'traceback': str(errorTraceback)
}
DEBUG('externalApi:Exception', errorData)
return jsonify(errorData), getattr(e, 'code', 500)
def parseIndexLine(s, full=False):
try:
parts = s.split('\t')
id = parts[0]
ip = parts[1]
timestamp = parts[2]
if full:
return {'id': id, 'ip': ip, 'timestamp': timestamp}
return id
except Exception as e:
error = str(e)
eTraceback = str(traceback.format_exc())
DEBUG('imageUtils:parseIndexLine: error', {
'error': error,
's': s,
'traceback': eTraceback,
})
raise Exception(
'Invalid images index line format: \'%s\' (parser message: %s)' %
(s, error))
DEBUG('imageUtils:removeImages: file not exist',
{'imageFilePath': imageFilePath})
else:
raise
try:
os.remove(yamlFilePath)
DEBUG('imageUtils:removeImages: deleted',
{'yamlFilePath': yamlFilePath})
except Exception, e:
if e.errno == errno.ENOENT:
DEBUG('imageUtils:removeImages: file not exist',
{'yamlFilePath': yamlFilePath})
else:
raise
DEBUG('imageUtils:removeImages: done', {
'ids': ids,
'imagesList': imagesList,
})
def removeAllImages():
"""
Remove all images
"""
uploadPath = config['uploadPath']
# filelist = [ file for file in os.listdir(mydir) if file.endswith('.bak') ] # TODO: Filter example
DEBUG('imageUtils:removeAllImages', {'uploadPath': uploadPath})
for file in os.listdir(uploadPath):
DEBUG('imageUtils:removeAllImages: remove file', {'file': file})
os.remove(os.path.join(uploadPath, file))
def getRecentImageData():
data = imageUtils.getRecentImageData()
DEBUG('externalApi:getRecentImageData called (/api/images/recent)', {
'data': data,
})
return jsonify(data)
'type': response.headers.type,
})
data = response.read()
DEBUG('htmlLoader: data readed', {
'url': url,
# 'data': utils.BlockString(data, 500),
'data': utils.truncateLongString(data, 600),
})
return data
except Exception, error:
DEBUG('htmlLoader: error catched', {
'url': url,
'error': errors.toBlockString(error),
})
__all__ = [ # Exporting objects...
'loadHtml',
]
if __name__ == '__main__': # Test
# url = 'https://echo.msk.ru/programs/radiodetaly/'
url = 'http://www.example.com/'
DEBUG('Testing for url', {'url': url})
html = loadHtml(url)
if html:
DEBUG('loaded html', {
'url': url,
'html': utils.BlockString(html, 300),
})
# -*- coding:utf-8 -*-
# @module removeImages
# @since 2020.10.01, 23:40
# @changed 2020.10.01, 23:40
import pathmagic # noqa # Add parent path to import paths for import config in debug mode
from logger import DEBUG
import imageUtils
def removeImages():
imageUtils.removeAllImages()
__all__ = [ # Exporting objects...
'removeImages',
]
if __name__ == '__main__':
result = removeImages()
DEBUG('test removeImages', {'result': result})
#!python2
# -*- coding:utf-8 -*-
# @module fetcher
# @since 2020.02.23, 18:51
# @changed 2020.02.23, 18:51
from logger import DEBUG
import htmlLoader
import htmlParser
def fetchMatchesFromUrl(url):
html = htmlLoader.loadHtml(url)
if html:
matches = htmlParser.fetchMatchesFromHtml(html)
return matches
__all__ = [ # Exporting objects...
'fetchMatchesFromUrl',
]
if __name__ == '__main__': # Test
url = 'https://echo.msk.ru/programs/radiodetaly/'
matches = fetchMatchesFromUrl(url)
DEBUG('fetcher: testing for fetchMatchesFromUrl', {'url': url, 'matches': matches})
# @changed 2020.02.23, 18:13
from os import path
from logger import DEBUG
import re
def fetchMatchesFromHtml(html):
matches = re.findall(r'href="([^"]*\.mp3)"', html)
# DEBUG('htmlParser:fetchMatchesFromHtml: found matches', {
# 'matches': matches,
# })
return matches
__all__ = [ # Exporting objects...
'fetchMatchesFromHtml',
]
if __name__ == '__main__': # Test
modulePath = path.dirname(path.abspath(__file__))
testFileName = 'parser-test-radiodetaly-fragment.html'
testFilePath = path.join(modulePath, testFileName)
print 'Testing with', testFilePath
with open(testFilePath) as file:
html = file.read()
matches = fetchMatchesFromHtml(html)
DEBUG('htmlParser:test:fetchMatchesFromHtml: found matches', {
'matches': matches,
})
fileSize = path.getsize(imageFilePath)
DEBUG(
'sendImageFile: image found', {
'imageFilePath': imageFilePath,
'yamlFilePath': yamlFilePath,
'fileSize': int(fileSize),
'params': params,
})
# params data sample:
# - ext: jpg
# - filename: test-image.jpg
# - id: 200930-225108
# - mimeType: image/jpeg
# - name: test-image
# - timestamp: 2020.09.30-22:51:08
return send_file(imageFilePath, mimetype=mimeType)
# return render_template('viewImage.html', id=id, timestamp=timestamp, imageWidth=imageWidth, imageHeight=imageHeight, params=params)
__all__ = [ # Exporting objects...
'listAllImages',
'viewImage',
'viewLastImage',
'sendImageFile',
]
if __name__ == '__main__':
# listAllImages()
result = viewImage('200930-225108')
DEBUG('test listImages', {'result': result})