def _find_files(root, includes, excludes, follow_symlinks):
"""List files inside a directory based on include and exclude rules.
This is a more advanced version of `glob.glob`, that accepts multiple
complex patterns.
Args:
root (str): base directory to list files from.
includes (list[str]): inclusion patterns. Only files matching those
patterns will be included in the result.
excludes (list[str]): exclusion patterns. Files matching those
patterns will be excluded from the result. Exclusions take
precedence over inclusions.
follow_symlinks (bool): If true, symlinks will be included in the
resulting zip file
Yields:
str: a file name relative to the root.
Note:
Documentation for the patterns can be found at
http://www.aviser.asia/formic/doc/index.html
"""
root = os.path.abspath(root)
file_set = formic.FileSet(
directory=root,
include=includes,
exclude=excludes,
symlinks=follow_symlinks,
)
for filename in file_set.qualified_files(absolute=False):
yield filename
def previewHtml():
server = Server()
for filepath in formic.FileSet(include=sourcedir,
exclude=[sourcedir + "/*_generated/**"]):
print("Watching: " + filepath)
server.watch(filepath, 'make html')
server.serve(root=builddir + 'html', open_url=False, host='0.0.0.0')
def _copy_files(self, staging_dir, includes, excludes):
package_files = formic.FileSet(includes, excludes)
for directory, file_name in package_files.files():
if directory:
target_path = os.path.join(staging_dir, directory, file_name)
create_dir(os.path.join(staging_dir, directory))
else:
target_path = os.path.join(staging_dir, file_name)
local('cp {0} {1}'.format(os.path.join(directory, file_name),
target_path))
def live():
"""Run livereload server"""
from livereload import Server
import formic
server = Server(app)
# css
for filepath in formic.FileSet(include="application/static/css/**/*.css"):
server.watch(filepath)
# html
for filepath in formic.FileSet(include="application/templates/css/**/*.html"):
server.watch(filepath)
# js
for filepath in formic.FileSet(include="application/static/js/**/*.js"):
server.watch(filepath)
# image
for filepath in formic.FileSet(include="application/static/image/**/*.*"):
server.watch(filepath)
server.serve(port=PORT)
def get_subtitle(id, file_name):
"""Fetches subtitles automatically using frameboise, then converts them to srt with subconvert"""
# Let's loop through our subtitle languages from config.py
for lang, value in SUBTITLES_LANG_PRIORITY.items():
# Let Framboise fetch the files one by one:
command = 'framboise "{!s}" -l {!s} --no-recursive'.format(
file_name, lang)
# Let's run the command and store the output
proc = subprocess.Popen(shlex.split(command), stdout=subprocess.PIPE)
result = proc.communicate()
# Get the subtitle filename
subtitle = ''.join(result[0])
subtitle = subtitle.split('\n')
# Subtitle was found? Add it to database fool
try:
if subtitle[0].find('Found subtitle') != -1:
subtitle = subtitle[2]
# F**k your whitespace
subtitle = subtitle.strip()
# Silly sub files, go away
filename, file_extension = os.path.splitext(subtitle)
if file_extension == '.sub':
sub2srt(subtitle)
subtitle = subtitle.replace('.sub', '.srt')
# Why do you not enforce .lang. in filename?!
if subtitle.find('.{!s}.'.format(lang)) == -1:
new_subtitle = subtitle.replace('.srt',
'.{!s}.srt'.format(lang))
os.rename(subtitle, new_subtitle)
subtitle = new_subtitle
# Don't need absolute path for database
subtitle = subtitle.replace(VIDEO_FOLDER, '')
# Fetch movie info
m = Movie.query.get(id)
# Insert subtitle to database
s = Subtitle(srt=subtitle, language=value, movie=m)
db.session.add(s)
db.session.commit()
except IndexError:
pass
# Oh hey where are we
path = os.path.abspath(file_name)
path = os.path.dirname(path)
# Let's search for those darn .sub files
fileset = formic.FileSet(include=["*.sub"], directory=path)
# F**k your .sub, .srt for life yo
for filename in fileset:
command = 'subconvert -q "{!s}"'.format(filename)
proc = subprocess.Popen(shlex.split(command))
proc.communicate()
def iter_paths_matching_substr(dir, substr=None, include=None, exclude=None, symlinks=True, include_origin=False):
substr = '*' if substr is None else f'*{substr}*'
substr = substr.replace('/', '*/**/*')
include = _parse_globs(include, substr, ['**/{}', '**/{}/**'])
exclude = _parse_globs(exclude, substr, '.*')
matches = sorted(
Path(x) for x in formic.FileSet(
directory=dir,
include=include,
exclude=exclude,
symlinks=symlinks,
)
)
if include_origin:
yield from ((dir, x) for x in matches)
else:
yield from matches
def scan_folders():
"""Scans video folders for video files, converts new videos and adds them to DB"""
movies = []
# Check for mp4, mkv, avi files in video folder
fileset = formic.FileSet(include=["*.mp4", "*.mkv", "*.avi"],
directory=VIDEO_FOLDER)
# Loop found files
for file_name in fileset.files():
file_name = os.path.join(fileset.get_directory(), file_name[1])
# Return relative path of found files
rec_name = file_name.replace(VIDEO_FOLDER, '')
rel_name = rec_name
rec_name = rec_name.replace('.avi', '.mp4')
# See if path found in database, if not: continue
if Movie.query.filter_by(url=rec_name).first() is None:
# or Movie.query.filter_by(url = rec_name).first() < 1:
# Check if filesize is different after 5 second, aka not completely uploaded yet
size = os.path.getsize(file_name)
time.sleep(5)
if size == os.path.getsize(file_name):
# Generate subtitle path from movie path
# FIXME: Not needed anymore I guess
subsrt = rec_name.replace('.mp4', '.srt')
subsrt = subsrt.replace('.avi', '.srt')
subsrt = subsrt.replace('.mkv', '.srt')
# Get extension and add movie to database as IN_PROGRESS
extension = os.path.splitext(rel_name)[1]
dbextension = os.path.splitext(rec_name)[1]
newmovie = Movie(url=rec_name, srt=subsrt, type=dbextension)
db.session.add(newmovie)
db.session.commit()
# Fetch subtitles automatically
get_subtitle(newmovie.id, file_name)
# Check if needs converting or can just call done
if extension == '.mp4':
complete(newmovie.id)
elif extension == '.avi':
convert_mp4(newmovie.id, file_name)
elif extension == '.mkv':
complete(newmovie.id)
def start(host, port):
app = Flask(__name__, static_folder='/static')
app.config['DEBUG'] = True
@app.route('/')
def index():
template_name = request.args.get('template')
vars_file = request.args.get('vars')
make_pdf = request.args.get('pdf')
if template_name is None:
return usage
if vars_file is not None:
with open(os.path.join('template-vars', vars_file), 'r') as f:
template_vars = json.load(f)
else:
template_vars = {}
for folder in jinja_config['template_folders']:
copy_tree(folder, '/static', update=1)
env = create_environment(jinja_config)
template = env.get_template(template_name)
rendered = template.render(**template_vars).encode('utf-8')
if make_pdf:
print('MAKING PDF!')
with open('/static/pdf_tmp.html', 'wb') as pdf:
pdf.write(rendered)
Popen(render_pdf_cmd, shell=True).wait()
return rendered
server = Server(app.wsgi_app)
watched_files = formic.FileSet(
include='**/*',
exclude=['templates/**/*.css', 'templates/pdf_tmp.html', '**/*.pdf'])
for file_name in watched_files:
server.watch(file_name, shell('node-sass-chokidar scss -o templates'))
server.serve(host=host, port=port, liveport=35729)
#!/usr/bin/env python
# install dependencies via `pip install -r watch_requirements.txt`
# install browser extension from http://livereload.com/extensions/
from __future__ import print_function
from livereload import Server, shell
import formic
EXCLUDES = ['**/vendor/**', '**/node_modules/**', '**/data/**']
UNCOMPILED_FILES = formic.FileSet(include=['**.css', '**.html', '**.md'],
exclude=EXCLUDES)
JS_APP_FILES = formic.FileSet(include=['**.js'],
exclude=EXCLUDES +
['/app.js', '/static/voterunner.js'])
COMBINED_JS_APP_FILES = formic.FileSet(include=['/static/voterunner.js'])
NODE_APP_FILES = formic.FileSet(include=['app.js'])
SCSS_FILES = formic.FileSet(include=['**.scss'], exclude=EXCLUDES)
CSS_FILES = formic.FileSet(include=['**.css'], exclude=EXCLUDES)
server = Server()
def watch(file_set, shell_command=None, delay=None):
for path in file_set:
server.watch(path, shell_command, delay=delay)
def show(file_set):
map(print, file_set)
conn.text_factory = str
# 9-ifInUcastPkts , 10-ifInErrors , 11-ifInDiscards , 13-ifOutUcastPkts , 14-ifOutErrors , 15-ifOutDiscards
cur.execute('''CREATE TABLE IF NOT EXISTS WirelessFailure
(id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, ifInUcastPkts INTEGER, ifInErrors INTEGER, ifInDiscards INTEGER, ifOutUcastPkts INTEGER, ifOutErrors INTEGER, ifOutDiscards INTEGER)'''
)
while True:
#path = '/Users/robinhood/aa-workspace-python/GMANE/fall03/031101/AcadBldg1AP1'
#files = glob.glob(path)
#walk the path through all the subdirectories
fileset = formic.FileSet(
include="**/*.gz",
directory="/Users/robinhood/aa-workspace-python/GMANE/fall03/")
for filename in fileset:
count = count + 1
print filename
with gzip.open(filename, 'rb') as f:
#file_content = f.read()
for line in f:
#print file_content
#while True:
# for line in file_content:
#countAllLines = countAllLines + 1
#print line
if line.startswith("c1"):
countC1 = countC1 + 1
# Checking root folder, this should be rewritten
dircheck = input(
'Is this script being run from the root directory of the LMST webserver (/opt/LMST/web by default)? (yes/no): '
)
if dircheck == "yes":
print('Continuing...')
elif dircheck == "no":
print('Exiting, rerun from the webserver root')
raise SystemExit(1)
else:
print('Invalid response, Exiting')
raise SystemExit(1)
# Glob all files with approved extensions
currentdir = os.getcwd() + '/Music/' #Default Music Directory
muspattern = ["*.mp3", "*.ogg", "*.opus"] # Default Extensions
filelist = formic.FileSet(directory=currentdir, include=muspattern)
musiclist = [] #Predefine Music List
for filename in filelist.qualified_files():
# Append Music File to List
musiclist.append(filename)
# Configure SQLite
connect = sqlite3.connect('lmst.db')
cursor = connect.cursor()
cursor.execute('DROP TABLE IF EXISTS song')
cursor.execute(
'CREATE TABLE song (id text,title text,artist text,album text,length text, genre text, path text)'
)
songindex = 0 # Variable for the song index in table
for fname in musiclist: # Start Scraping info from songs and append to DB
songindex = songindex + 1
# Parsing the command and running GREP appears to need multiple pipes, look into fixing this later
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project.settings")
from django.core.management import execute_from_command_line
from dj_static import Cling, MediaCling
from local_settings import ENV
if len(sys.argv) >= 2 and sys.argv[1] == 'livereload':
import formic
from django.core.wsgi import get_wsgi_application
from livereload import Server
application = get_wsgi_application()
server = Server(Cling(application))
# Add your watch
for filepath in formic.FileSet(include=["**/*.html", "**/*.js"]):
server.watch(filepath)
server.serve(port=8000)
else:
execute_from_command_line(sys.argv)
