def _generate_cert(self):
if not os.path.exists(self.cert_dir):
os.makedirs(self.cert_dir)
t = tpl.render(common_name = self.cn, ca_cert_dir = CA_DIR )
fh = tempfile.NamedTemporaryFile(delete=False)
fh.write(t)
fh.close()
ssl_conf = fh.name
key = os.path.join(self.cert_dir, 'cert.key')
csr = os.path.join(self.cert_dir, 'cert.csr')
crt = os.path.join(self.cert_dir, 'cert.crt')
success = True
#Now fiddle with all the shitty options of openssl(1)
try:
lock.acquire()
if not os.path.exists(key):
cmd = 'openssl req -config %s -days 365 -nodes -new -keyout %s -out %s' % (ssl_conf, key,csr)
self.log.error(cmd)
gencert = envoy.run(cmd)
if gencert.status_code != 0:
raise OSError(gencert.status_code, "problem generating the certificate: %s" % gencert.std_err)
if not os.path.exists(crt):
cmd = 'openssl ca -batch -notext -config %s -out %s -infiles %s' % (ssl_conf, crt, csr)
self.log.error(cmd)
signcert = envoy.run(cmd)
if signcert.status_code != 0:
raise OSError(signcert.status_code, "problem signing the certificate: %s" % signcert.std_err)
if not os.path.exists(self.cert_path):
destination = open(self.cert_path, 'wb')
shutil.copyfileobj(open(crt, 'rb'), destination)
shutil.copyfileobj(open(key, 'rb'), destination)
destination.close()
except Exception, e:
self.log.error(str(e))
success = False
def run(self):
# stop Gammu-SMSD
envoy.run('sudo /etc/init.d/gammu-smsd stop')
environ = dict(os.environ)
def target():
self.process = subprocess.Popen(self.cmd,
universal_newlines=True,
shell=False,
env=environ,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=0,
)
self.out, self.err = self.process.communicate()
self.thread = threading.Thread(target=target)
self.thread.start()
self.running = True
self.timer = self.startTimer(self.loop * 1000)
def demo():
import envoy
import heroku
user = request.values['user'].encode("utf-8")
repo = request.values['repo'].encode("utf-8")
username = os.environ['HEROKU_USER']
password = os.environ['HEROKU_PASS']
cloud = heroku.from_pass(username, password)
app = cloud.apps.add()
r = envoy.run("git clone %s %s" % (repo, app.name.encode("utf-8")), timeout=15)
try:
os.chdir(app.name.encode("utf-8"))
print 1
r = envoy.run("git remote add heroku [email protected]:%s.git" % app.name.encode("utf-8"))
print 2
r = envoy.run('git push heroku master')
print 3
app = cloud.apps[app.name]
print 4
app.collaborators.add(user)
print 5
app.transfer(user)
print 6
result = Response(response = json.dumps({'result': "success", 'name': app.name}), mimetype="application/json")
print 7
except Exception, e:
print e
app.destroy()
result = Response(response = json.dumps({'result': "failed"}), mimetype="application/json")
def _create_jp2_from_tif( self, CONVERT_COMMAND_PATH, KAKADU_COMMAND_PATH, source_filepath, destination_filepath ):
""" Creates jp2 directly. """
cleaned_source_filepath = source_filepath.replace( ' ', '\ ' )
cmd = '%s -i "%s" -o "%s" Creversible=yes -rate -,1,0.5,0.25 Clevels=12' % (
KAKADU_COMMAND_PATH, cleaned_source_filepath, destination_filepath )
self.logger.info( 'in image._create_jp2_from_tif(); cmd, %s' % cmd )
r = envoy.run( cmd.encode('utf-8', 'replace') ) # envoy requires a non-unicode string
self.logger.info( 'in image._create_jp2_from_tif(); r.std_out, %s' % r.std_out )
self.logger.info( 'in image._create_jp2_from_tif(); r.std_err, %s' % r.std_err )
if 'compressed TIFF' in r.std_err: # kakadu can't handle that
## uncompress the tiff
cmd = '%s -compress None "%s" %s' % (
CONVERT_COMMAND_PATH, cleaned_source_filepath, cleaned_source_filepath ) # replaces existing file with uncompressed version
self.logger.info( 'in image._create_jp2_from_tif(); compressed-tiff-handling cmd, %s' % cmd )
r = envoy.run( cmd.encode('utf-8', 'replace') ) # envoy requires a non-unicode string
self.logger.info( 'in image._create_jp2_from_tif(); compressed-tiff-handling r.std_out, %s' % r.std_out )
self.logger.info( 'in image._create_jp2_from_tif(); compressed-tiff-handling r.std_err, %s' % r.std_err )
## try kakadu again
cmd = '%s -i "%s" -o "%s" Creversible=yes -rate -,1,0.5,0.25 Clevels=12' % (
KAKADU_COMMAND_PATH, cleaned_source_filepath, destination_filepath )
self.logger.info( 'in image._create_jp2_from_tif(); try2 cmd, %s' % cmd )
r = envoy.run( cmd.encode('utf-8', 'replace') ) # envoy requires a non-unicode string
self.logger.info( 'in image._create_jp2_from_tif(); try2 r.std_out, %s' % r.std_out )
self.logger.info( 'in image._create_jp2_from_tif(); try2 r.std_err, %s' % r.std_err )
return
def db_insert(dt, interval):
db = connect_db()
#TODO - check for this filename's existence in DB
timestamp = datetime.now().strftime("%Y%m%d%H%M%s")
fpath = '%s/%s/%s.avi' % (VIDDIR, interval, dt)
fsize = os.stat(fpath).st_size
#TODO - prettier tile e.g. March 1, 2014
COLUMNS = "(title, filename, fullpath, interval, datetimestamp, notable, size)"
VALUES = "('%s', '%s.avi', '%s', '%s', '%s', 0, '%s')" % \
(dt, dt, fpath, interval, timestamp, fsize)
query = "insert into tl_videos %s values %s" % (COLUMNS, VALUES)
db.execute(query)
db.commit()
db.close()
# Restart system if video size is below threshold
if interval == 'hour' and fsize < 5000000:
print "Video size less than 5 MB which likely indicates a problem. Restarting system."
email("Timelapse - Restarting Server",
"The video " + os.path.basename(fpath) + " was only " + str(fsize) + " bytes. " + \
"This likely indicates a problem with the capture device over the past hour. " + \
"Restarting " + socket.gethostname())
time.sleep(10)
envoy.run("sudo reboot")
def main():
arguments = docopt.docopt(__doc__, version='urls 0.1')
if arguments['--debug']:
print(arguments)
theText = arguments['<text>']
if not theText:
theText = sys.stdin.read()
theURLs = URLsForString(theText, twitter = bool(arguments['--twitter']))
if not theURLs:
if not theText:
sys.stderr.write('# No URLs found on standard input\n')
else:
sys.stderr.write('# No URLs found\n')
else:
if not arguments['--no-output']:
print '\n'.join(set(theURLs))
if arguments['--copy']:
theCommand = 'echo \'%s\' | pbcopy' % theURLs[0]
envoy.run(theCommand)
if arguments['--open']:
envoy.run('open \'%s\'' % theURLs[0])
def download_repo(url, pull_if_exists=True):
url, username, repo = transform_url(url)
path = os.path.join(settings.REPO_ROOT, username, repo)
if os.path.exists(os.path.join(path, '.failed')):
return None
if os.path.exists(path):
if pull_if_exists:
with cd(path):
response = envoy.run('git checkout master')
if response.status_code != 0:
raise RuntimeError('Bad status_code from git checkout master: {0}. Git\'s stderr: {1}'.format(
response.status_code, response.std_err))
response = envoy.run('git pull')
if response.status_code != 0:
raise RuntimeError('Bad status_code from git pull: {0}. Git\'s stderr: {1}'.format(
response.status_code, response.std_err))
else:
response = envoy.run('git clone {url} {path}'.format(url=url, path=path))
if response.status_code != 0:
os.makedirs(path)
with open(os.path.join(path, '.failed'), 'w') as f:
f.write('')
raise RuntimeError('Bad status_code from git clone: {0}. Git\'s stderr: {1}'.format(
response.status_code, response.std_err)
)
return path
def restart(self):
"""
Restarts the BTS. Note that OpenBTS must be running as a supervisorctl
job.
"""
logging.warning("Restarting openbts")
envoy.run("sudo supervisordctl restart openbts")
def create_archive(self):
''' creates tarball
'''
dump_directory = os.path.join(self.working_directory, 'dump/')
self.archive_directory = os.path.join(self.working_directory
, 'dump.tgz')
envoy.run('tar -czf %s %s' % (self.archive_directory, dump_directory))
def on_modified(self, event):
global WATCH, LASTRUN
what = 'directory' if event.is_directory else 'file'
if what == 'file' and event.src_path == './%s' % WATCH and time.time()-LASTRUN > 1.0:
LASTRUN = time.time()
logging.info("Modified %s: %s", what, event.src_path)
envoy.run('pkill -x -f "python %s"' % WATCH)
proc = subprocess.Popen(['python 007-vispy-tutorial.py'], shell=True, stdin=None, stdout=None, stderr=None, close_fds=True)
def import_json_to_mongo():
r = envoy.run([MONGO_DIR + 'mongoimport'])
print r.std_out
if r.std_err: print r.std_err
data_file = os.path.join(os.getcwd(), 'data\\enron.mbox.json')
r = envoy.run([MONGO_DIR + 'mongoimport --host localhost --db enron --collection mbox --file "%s"' % data_file])
print r.std_out
if r.std_err: print sys.stderr.write(r.std_err)
def set_new_neighbor_list(self, neighbors):
neighbor_string = " ".join([str(_) for _ in neighbors])
# THIS IS THE OFFICIAL WAY TO DO THIS
# IN THE NAME OF ALL THAT IS HOLY
envoy.run("echo -n 'config GSM.CellSelection.Neighbors %s' | sudo /OpenBTS/OpenBTSDo" % neighbor_string)
# ignore measurement requests for a while
logging.info("New neighbor list: %s" % neighbor_string)
def every_notify(self, *args, **kwargs):
""" fetch new cap"""
irc_c = args[0].get("client").irc_c
w = Walrus()
notify = w.List("notify")
while len(notify)>0:
m = notify.pop()
irc_c.PRIVMSG(self.chan, m)
envoy.run("rrdtool-nginx.pl")
def main():
parser = argparse.ArgumentParser(description=description, epilog=epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('-c', '--collection', required=True, help='Absolute path to source collection repository.')
parser.add_argument('-d', '--debug', action='store_true', help="Don't change anything, just list files that would be changed.")
parser.add_argument('-m', '--move', action='store_true', help="Don't just copy logfiles, move them to new dir.")
parser.add_argument('-r', '--rmtmp', action='store_true', help="Remove raw logfiles after finishing tgz.")
args = parser.parse_args()
collection_path = os.path.realpath(args.collection)
debug = args.debug
if args.move:
cpmv = 'mv'
else:
cpmv = 'cp'
media_base = os.path.dirname(collection_path)
collection_id = os.path.basename(collection_path)
logs_dir = os.path.join(media_base, 'log')
dest_dir = os.path.join(media_base, 'log', '%s-addfiles' % collection_id)
tgz_path = os.path.join('%s.tgz' % dest_dir)
print('media_base %s' % media_base)
print('dest_dir %s' % dest_dir)
print('tgz_path %s' % tgz_path)
if not os.path.exists(dest_dir):
print('making dest dir %s' % dest_dir)
if not debug:
os.makedirs(dest_dir)
# copy files
print('copying...')
logfiles = find_logfiles(collection_path)
for logpath in logfiles:
entity_id = os.path.basename(os.path.dirname(logpath))
dest = os.path.join(dest_dir, '%s.log' % entity_id)
cmd = '%s %s %s' % (cpmv, logpath, dest)
print(cmd)
if not debug:
r = envoy.run(cmd)
# tgz
print('compressing...')
os.chdir(logs_dir)
TAR_CMD = 'tar czvf %s %s' % (os.path.basename(tgz_path), os.path.basename(dest_dir))
print(TAR_CMD)
if not debug:
r = envoy.run(TAR_CMD)
print(r.std_out.strip())
print('-> %s' % tgz_path)
if args.rmtmp:
print('removing temp files %s' % dest_dir)
if not debug:
shutil.rmtree(dest_dir)
def main():
if DEBUG:
print 'Entering debug mode...'
check_times = {}
# hash sound filenames, detect type, retrieve if necessary
for (stop_id, sound_file) in STOPS_AND_THEIR_SOUND_FILES.items():
STOPS_AND_THEIR_SOUND_FILES[stop_id] = {'sound_file': sound_file}
STOPS_AND_THEIR_SOUND_FILES[stop_id]['hash'] = hashlib.sha256(sound_file).hexdigest()
STOPS_AND_THEIR_SOUND_FILES[stop_id]['type'] = sound_file.split('.')[-1].lower().strip()
STOPS_AND_THEIR_SOUND_FILES[stop_id]['path'] = "%s/sounds/%s.%s" % (WORKING_DIRECTORY, STOPS_AND_THEIR_SOUND_FILES[stop_id]['hash'], STOPS_AND_THEIR_SOUND_FILES[stop_id]['type'])
if not os.path.exists(STOPS_AND_THEIR_SOUND_FILES[stop_id]['path']):
if DEBUG:
print 'retrieving %s' % (sound_file)
envoy.run('curl "%s" -o "%s"' % (sound_file, STOPS_AND_THEIR_SOUND_FILES[stop_id]['path']))
check_times[stop_id] = 0
while True:
for (stop_id, ct) in check_times.items():
played_sound = False
if time.time()>ct:
if DEBUG:
print 'checking times for stop %s' % stop_id
predictions = None
while predictions is None:
try:
predictions = nextbus2.get_predictions_for_stop(AGENCY_TAG, stop_id).predictions
except:
time.sleep(60)
if len(predictions)==0:
check_times[stop_id] = time.time() + (10*60)
else:
shortest_time = predictions[0].seconds
for p in predictions:
if p.seconds<PLAY_SOUND_IF_BUS_IS_LESS_THAN_THIS_MANY_SECONDS_AWAY:
play_sound(stop_id)
played_sound = True
shortest_time = min(shortest_time, p.seconds)
if shortest_time>180:
check_times[stop_id] = time.time() + (shortest_time/2)
else:
if played_sound:
check_times[stop_id] = time.time() + (PLAY_SOUND_IF_BUS_IS_LESS_THAN_THIS_MANY_SECONDS_AWAY * 2)
else:
check_times[stop_id] = time.time() + 30
if DEBUG:
print 'will check %s again in %d seconds' % (stop_id, (check_times[stop_id] - time.time()))
time.sleep(1)
def putfile(self, fname, contents):
fbuf = '/tmp/buf.%d' % os.getpid() # XXX
uwsgi = open(fbuf, 'w')
uwsgi.write(contents)
uwsgi.close()
envoy.run('sudo mv %s %s' % (fbuf, fname))
def update(ref):
branch = None
if ref == 'refs/heads/master':
branch = 'master'
if ref == 'refs/heads/dev':
branch = 'dev'
if branch is None:
return
envoy.run("./update.sh %s" % branch)
def set_permissions(drive):
if 'linux' in sys.platform:
print "Setting permissions on", drive
logger.info("\nSetting permissions on drive ...\n")
p = envoy.run('chown -R root:root /media/'+drive)
p = envoy.run('chmod -R 664 /media/'+drive)
p = envoy.run('chmod -R +X /media/'+drive)
p = envoy.run('find /media/'+drive+ ' -name "*.sh" | xargs chmod +x')
else:
print "Cannot set PERMISSIONS for " + drive + ". Unknown OS: " + sys.platform
def play_sound(stop_id):
if DEBUG:
print 'playing sound for %s' % stop_id
try:
if STOPS_AND_THEIR_SOUND_FILES[stop_id]['type']=='mp3':
envoy.run('/usr/bin/mpg321 %s' % (STOPS_AND_THEIR_SOUND_FILES[stop_id]['path']))
if STOPS_AND_THEIR_SOUND_FILES[stop_id]['type']=='wav':
envoy.run('/usr/bin/aplay %s' % (STOPS_AND_THEIR_SOUND_FILES[stop_id]['path']))
except Exception, e:
print 'attempted to play %s' % STOPS_AND_THEIR_SOUND_FILES[stop_id]['path']
def restart_openbts(self):
""" TODO OpenBTS should really be a service, and we should really just say
"sudo service restart openbts". But we can't, because OpenBTS is a disaster.
EVEN WORSE, we assume that we're in OpenBTS's runloop which will restart us
automatically. What a mess... """
logging.warning("Restarting OpenBTS...")
envoy.run("killall %s %s" % (OPENBTS_PROCESS_NAME, TRANSCEIVER_PROCESS_NAME))
time.sleep(10)
if len(envoy.run("ps aux | grep './OpenBTS'"))==0:
pass # TODO: run OpenBTS
def handle_one_test(testname):
print(str(failed_test.group('testname')))
try:
os.makedirs("./_test-identify/%s" % (testname))
except:
pass
envoy.run("%s %s" % (FAUP_TEST_CMD, testname))
shutil.copyfile(URLS_ORIGIN, "./_test-identify/%s/%s" % (testname, "urls.txt.ref"))
shutil.copyfile(URLS_CMP, "./_test-identify/%s/%s" % (testname, "urls.txt.cmp"))
shutil.copyfile(URLS_FILE, "./_test-identify/%s/%s" % (testname, "urls.txt"))
def adsl(self):
"""
"""
env = {
"PATH" : "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games",
"SHELL" : "/bin/bash",
"USER" : "root",
"PWD" : "/root/work/dail",
"_" : "/usr/bin/env",
"OLDPWD" : "/root/work",
"XDG_RUNTIME_DIR" : "/run/user/0",
"LESSOPEN" : "| /usr/bin/lesspipe %s",
"SHLVL" : "1",
}
while True:
print('ADSL Start, Remove Proxy, Please wait')
self.remove_proxy()
r = envoy.run(ADSL_BASH_STOP)
if r.status_code == 0:
print('stop success')
else:
print(r.std_out)
print(r.std_err)
time.sleep(ADSL_ERROR_CYCLE)
continue
s = envoy.run(ADSL_BASH_START)
if s.status_code == 0:
print('start success')
else:
print(s.std_out)
print(s.std_err)
time.sleep(ADSL_ERROR_CYCLE)
continue
print('ADSL Successfully')
ip = self.get_ip()
if ip:
print('Now IP', ip)
print('Testing Proxy, Please Wait')
proxy = '{ip}:{port}'.format(ip=ip, port=PROXY_PORT)
if self.test_proxy(proxy):
print('Valid Proxy')
self.set_proxy(proxy)
print('Sleeping')
time.sleep(ADSL_CYCLE)
else:
print('Invalid Proxy')
else:
print('Get IP Failed, Re Dialing')
time.sleep(ADSL_ERROR_CYCLE)
def launch(url, port=1337, profile_dir=os.path.expanduser(os.path.join("~", "chrome-remote-profile")), canary=False):
"""Launch an instance of Google Chrome for remote debugging."""
app = "Google Chrome"
if canary:
app += " Canary"
chrome = os.path.join(os.sep, "Applications", app + os.path.extsep + "app", "Contents", "MacOS", app)
assert os.path.exists(chrome), "'%s' does not exist." % chrome
chrome = chrome.replace(" ", "\ ")
cmd = "'%s' --remote-debugging-port=%d --user-data-dir=%s %s" % (chrome, port, profile_dir, url or "")
envoy.run(cmd)
def main():
if os.path.exists('wonderfan.github.io'):
envoy.run("rm -rf wonderfan.github.io")
repo = '[email protected]:wonderfan/wonderfan.github.io.git'
envoy.run('git clone '+ repo)
cwd = "wonderfan.github.io"
get_screenshot()
envoy.run("git add --all ",cwd=cwd)
envoy.run('git commit -m "submit the changes"',cwd=cwd)
envoy.run("git push origin master",cwd=cwd)
def git_commits():
"""Returns various repos' most recent Git commit.
"""
commits = {}
pub_path = os.getcwd()
cmd_path = settings.CMDLN_INSTALL_PATH
def_path = settings.REPO_MODELS_PATH
cmd = 'git log --pretty=format:"%h %ci%d" -1'
os.chdir(pub_path); commits['pub'] = envoy.run(cmd).std_out
os.chdir(cmd_path); commits['cmd'] = envoy.run(cmd).std_out
os.chdir(def_path); commits['def'] = envoy.run(cmd).std_out
return commits
def list(self, ignore_patterns):
for path, storage in super(PrecompilerFinder, self).list(ignore_patterns):
file_name, file_ext = os.path.splitext(path)
if file_ext in settings.STATICFILES_PRECOMPILERS:
handler = settings.STATICFILES_PRECOMPILERS[file_ext]
new_path = handler[1].format(file_name=file_name)
infile = storage.path(path)
outfile = storage.path(new_path)
envoy.run(handler[0].format(infile=infile, outfile=outfile))
yield new_path, storage
else:
yield path, storage
def cloneRepos(shell, hostname):
logging.info('Cloning git repositories to instance')
checkRequiredSettings(['git_repos_to_clone', 'git_ssh_key'])
# Check that Git is installed
shell.run(shlex.split('sudo apt-get install git'))
# Copy SSH key to server
shell.run(shlex.split('mkdir -p ~/.ssh/'))
envoy.run('scp -i %s %s@%s:.ssh/.' % (settings['git_ssh_key'], settings['ami_default_username'], hostname))
for repoUrl in settings['git_repos_to_clone']:
shell.run(shlex.split('git clone %s' % repoUrl))
def clean(f="setup.py"):
"""python setup.py clean --all"""
f=abspath(expanduser(f))
pwd = getcwd()
if _exists(f):
try:
chdir(dirname(f))
command="python setup.py clean --all"
run(command)
chdir(pwd)
except Exception,e:
chdir(pwd)
raise Exception(str(e))
def main(args):
""" the body of the script
"""
tmpdir = mkdtemp()
if args.file.startswith(('http://', 'https://')):
print "The file is in the net, downloading it..."
file_basename = os.path.basename(
urllib2.urlparse.urlsplit(args.file).path
)
result = envoy.run('wget "{}" -O {}'.format(args.file, file_basename))
if result.status_code:
print_error_result(
result, "Error while downloading RDF data {}. Aborting".format(
args.file
)
)
exit(1)
filename = os.path.join(
tmpdir, file_basename
)
else:
print "The file is local, moving it..."
shutil.copy(args.file, tmpdir)
filename = os.path.join(tmpdir, os.path.basename(args.file))
print "handling file", filename
filename_cropped, extension = os.path.splitext(filename)
if extension == '.bz2':
print "Got a bz2 file, need to convert it with gzip"
gzip_filename = filename_cropped + '.gz'
result = envoy.run('bunzip2 "{}" -c | gzip > "{}'.format(
filename, gzip_filename
))
if result.status_code:
print_error_result(result, "Error while converting file, aborting")
exit(2)
filename = gzip_filename
print "File converted successfully, now handling", filename
print "Ingesting file in virtuoso"
virtuoso = get_virtuoso()
virtuoso.clear(args.graph)
print "Ingestion completed", virtuoso.ingest(filename, graph=args.graph)
def inline():
if self.servers:
bootstrap = '-bootstrap="%s:%d"' % (SERVER, self.servers.keys()[0])
else:
bootstrap = ""
if replication_port is None:
resp = envoy.run('go run ../main.go -block-server="%s:%d" %s' % (SERVER, port, bootstrap))
else:
resp = envoy.run('go run ../main.go \
-block-server="%s:%d" %s -replication=true \
-replication-server="%s:%d"' % (SERVER, port, bootstrap, SERVER, replication_port))
print resp.std_out
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
import envoy
output_file_str = './bin/gf_crawl_dashboard.js'
files_lst = [
'./src/dashboard/gf_crawl__img_preview_tooltip.ts',
'./src/dashboard/gf_crawl_dashboard.ts',
'./src/dashboard/gf_crawl_events.ts',
'./src/dashboard/gf_crawl_images_browser.ts',
]
print 'files_lst - %s' % (files_lst)
print 'RUNNING COMPILE...'
r = envoy.run('tsc --out %s %s' % (output_file_str, ' '.join(files_lst)))
print r.std_out
print r.std_err
def load_fixtures():
"""use test settings, not the production settings"""
# create a super-user who can create authorizations
# create_authz_superuser()
envoy.run(
'mongorestore --db test_dlkit_assessment --drop tests/functional/fixtures/test_dlkit_assessment'
)
envoy.run(
'mongorestore --db test_dlkit_authorization --drop tests/functional/fixtures/test_dlkit_authorization'
)
envoy.run(
'mongorestore --db test_dlkit_grading --drop tests/functional/fixtures/test_dlkit_grading'
)
envoy.run(
'mongorestore --db test_dlkit_learning --drop tests/functional/fixtures/test_dlkit_learning'
)
envoy.run(
'mongorestore --db test_dlkit_logging --drop tests/functional/fixtures/test_dlkit_logging'
)
envoy.run(
'mongorestore --db test_dlkit_repository --drop tests/functional/fixtures/test_dlkit_repository'
)
envoy.run(
'mongorestore --db test_dlkit_resource --drop tests/functional/fixtures/test_dlkit_resource'
)
def test_input(self):
r = envoy.run("sed s/i/I/g", "Hi")
self.assertEqual(r.std_out.rstrip(), "HI")
self.assertEqual(r.status_code, 0)
p = argparse.ArgumentParser(description='Generate a Certificate Signing Request')
p.add_argument('--new', dest='new', action="store_const",
const=True, default=False, help='Generate new UUID')
args = p.parse_args()
iden = None
if (args.new):
iden = uuid.uuid4()
else:
iden = "84bc2563-2da1-4924-91cc-86b4d19ebf55"
raw_input("Register BTS %s into the web interface. Press enter when done." % iden)
params = {
'bts_uuid': iden
}
r = requests.get("http://127.0.0.1:8080/api/v1/bts/sslconf", params=params)
if (r.status_code != 200):
raise Exception("Unable to generate ssl conf")
sslconf = json.loads(r.text)['sslconf']
with open("sslconf.conf", "w") as f:
f.write(sslconf)
envoy.run("openssl req -new -newkey rsa:2048"
" -config sslconf.conf"
" -keyout client.key"
" -out client.req -nodes")
def main(chapters=[],
epub=False,
pdf=False,
html=False,
mobi=False,
pandoc_epub=False):
if not os.path.isdir('output'):
os.mkdir('output')
else:
run('rm output/*')
chapter_dirs = [
'chrome',
'ethercalc',
'ninja',
'pugixml',
'memshrink',
'dance',
'infinispan',
'talos',
'zotonic',
'mobile-perf',
'warp',
'khmer',
]
if len(chapters) > 0:
chapter_dirs = [
chapter_dir for chapter_dir in chapter_dirs
if chapter_dir in chapters
]
chapter_markdowns = [
'./' + chapter_dir + '/' + chapter_dir + '.markdown'
for chapter_dir in chapter_dirs
]
chapter_markdowns_exist = [
envoy.run('test -f ' + chapter_markdown).status_code
for chapter_markdown in chapter_markdowns
]
process_chapters = [
chapter_markdown for chapter_markdown, process in zip(
chapter_markdowns, chapter_markdowns_exist) if process == 0
]
chapter_names = [getbasename(chapter) for chapter in chapter_dirs]
image_paths = [
'./ethercalc/ethercalc-images',
'./warp/warp-images',
'./zotonic/zotonic-images',
'./talos/talos-images',
'./chrome/chrome-images',
'./mobile-perf/mobile-perf-images',
'./pugixml/pugixml-images',
'./infinispan/infinispan-images',
'./khmer/khmer-images',
'./memshrink/memshrink-images',
'./dance/dance-images',
]
#run('cp /home/tavish/aosa/posa/dance/chapter.tex /home/tavish/aosa/posa/tex/dance.tex')
#[run("cp /home/tavish/aosa/posa/dance/{f} /home/tavish/aosa/posa/tex/dance{f}".format(f=c)) for c in run('ls ../dance | grep tex$ | grep -v chapter.tex').std_out.splitlines()]
with open('tex/posa.tex', 'w') as out:
with open('tex/posa.template.tex') as template:
lines = template.readlines()
for line in lines:
if 'chapterchapterchapter' in line:
out.write("\n".join("\include{%s}\n" % (chapter_name)
for chapter_name in chapter_names))
else:
out.write(line)
if pdf:
for imgpath in image_paths:
# This is silly but oh well
run("cp -a {imgpath} tex/".format(imgpath=imgpath))
texify_chapters = [
pandoc_cmd(chapter_markdown)
for chapter_markdown in process_chapters
]
build_pdf()
if epub:
for imgpath in image_paths:
# This is silly but oh well
run("cp -a {imgpath} epub/".format(imgpath=imgpath))
build_epub(process_chapters, pandoc_epub)
if mobi and not epub:
print "Can't build .mobi; depends on .epub."
print "Use --epub --mobi to build .mobi file."
elif mobi:
build_mobi()
if html:
for imgpath in image_paths:
# This is silly but oh well
run("cp -a {imgpath} html/content/pages/".format(imgpath=imgpath))
build_html(process_chapters)
for imgpath in image_paths:
# So you thought that was silly? Lemme show ya
run("cp -a {imgpath} html/output/pages/".format(imgpath=imgpath))
def blat(database, query, blatout):
print 'Running Blat, this may take a while...'
r = envoy.run('/public/software/blat/blat {d} {q} {o}'.format(q=query,
d=database,
o=blatout))
return r.std_err
def setup_method(self, method):
self.tmp_dir = tempfile.mkdtemp(prefix='datagristle_profiler_')
recs = [['Alabama', '8', '18'], ['Alaska', '6', '16'],
['Arizona', '6', '14'], ['Arkansas', '2', '12'],
['California', '19', '44'], ['Colorado', '19', '44'],
['Illinois', '19', '44'], ['Indiana', '19', '44'],
['Kansas', '19', '44'], ['Kentucky', '19', '44'],
['Louisiana', '19', '44'], ['Maine', '19', '44'],
['Mississippi', '19', '44'], ['Nebraska', '19', '44'],
['Oklahoma', '19', '44'], ['Tennessee', '19', '44'],
['Texas', '19', '9999'], ['Virginia', '19', '44'],
['West Virginia', '19', '44']]
self.file_struct = {}
self.field_struct = {}
fqfn = generate_test_file(delim='|',
rec_list=recs,
quoted=False,
dir_name=self.tmp_dir)
cmd = '%s \
--infiles %s \
--max-freq 10 \
--output-format=parsable' % (os.path.join(
script_path, 'gristle_profiler'), fqfn)
runner = envoy.run(cmd)
print(runner.std_out)
print(runner.std_err)
assert runner.status_code == 0
mydialect = csv.Dialect
mydialect.delimiter = '|'
mydialect.quoting = csvhelper.get_quote_number('QUOTE_ALL')
mydialect.quotechar = '"'
mydialect.lineterminator = '\n'
csvobj = csv.reader(runner.std_out.split('\n'), dialect=mydialect)
for record in csvobj:
if not record:
continue
if len(record) != 5:
if 'WARNING: freq dict is too large' in record[0]:
continue # ignore warning row
else:
pytest.fail('Invalid result record: %s' % record[0])
division = record[0]
section = record[1]
subsection = record[2]
key = record[3]
value = record[4]
assert division in [
'file_analysis_results', 'field_analysis_results'
]
if division == 'file_analysis_results':
assert section == 'main'
assert subsection == 'main'
self.file_struct[key] = value
elif division == 'field_analysis_results':
assert 'field_' in section
assert subsection in ['main', 'top_values']
if section not in self.field_struct:
self.field_struct[section] = {}
if subsection not in self.field_struct[section]:
self.field_struct[section][subsection] = {}
self.field_struct[section][subsection][key] = value
def main(chapters=[],
epub=False,
pdf=False,
html=False,
mobi=False,
pandoc_epub=False):
if not os.path.isdir('output'):
os.mkdir('output')
else:
output_files = glob.glob('output/*')
for f in output_files:
run('rm {}'.format(f))
chapter_dirs = [
'template-engine',
'pedometer',
'sampler',
'spreadsheet',
'cluster',
'data-store',
'objmodel',
'ci',
'crawler',
'modeller',
]
if len(chapters) > 0:
chapter_dirs = [
chapter_dir for chapter_dir in chapter_dirs
if chapter_dir in chapters
]
chapter_markdowns = [
'./' + chapter_dir + '/' + chapter_dir + '.markdown'
for chapter_dir in chapter_dirs
]
chapter_markdowns_exist = [
envoy.run('test -f ' + chapter_markdown).status_code
for chapter_markdown in chapter_markdowns
]
process_chapters = [
chapter_markdown for chapter_markdown, process in zip(
chapter_markdowns, chapter_markdowns_exist) if process == 0
]
chapter_names = [getbasename(chapter) for chapter in chapter_dirs]
image_paths = [
'./pedometer/pedometer-images',
'./sampler/sampler-images',
'./spreadsheet/spreadsheet-images',
'./cluster/cluster-images',
'./data-store/data-store-images',
'./objmodel/objmodel-images',
'./ci/ci-images',
'./crawler/crawler-images',
'./modeller/modeller-images',
]
run('cp -r minutiae/ tex')
with open('tex/500L.tex', 'w') as out:
with open('tex/500L.template.tex') as template:
lines = template.readlines()
for line in lines:
if 'chapterchapterchapter' in line:
out.write('\n'.join('\include{%s}\n' % (chapter_name)
for chapter_name in chapter_names))
else:
out.write(line)
if pdf:
for imgpath in image_paths:
# This is silly but oh well
run('cp -a {imgpath} tex/'.format(imgpath=imgpath))
for chapter_markdown in process_chapters:
pandoc_cmd(chapter_markdown)
build_pdf()
if epub:
for imgpath in image_paths:
# This is silly but oh well
run('cp -a {imgpath} epub/'.format(imgpath=imgpath))
build_epub(process_chapters, pandoc_epub)
if mobi and not epub:
print 'Cannot build .mobi; depends on .epub.'
print 'Use --epub --mobi to build .mobi file.'
elif mobi:
build_mobi()
if html:
for imgpath in image_paths:
# This is silly but oh well
run('cp -a {imgpath} html/content/pages/'.format(imgpath=imgpath))
build_html(process_chapters)
for imgpath in image_paths:
# So you thought that was silly? Lemme show ya
run('cp -a {imgpath} html/output/pages/'.format(imgpath=imgpath))
def run(cmd):
print cmd
result = envoy.run(cmd)
print result.std_out
print result.std_err
return result
pass
spyral.director.init(options.res,
fullscreen=options.fullscreen,
max_fps=options.fps)
launch()
def run():
spyral.director.run(profiling=True)
while spyral.director.get_scene():
scene = spyral.director.get_scene().__class__.__name__
scenes[scene] += 1
output = '%s/%s-%d' % (output_dir, scene, scenes[scene])
try:
cProfile.run('run()', '%s.pstats' % (output, ))
except KeyboardInterrupt:
spyral.quit()
files.append(output)
spyral.quit()
try:
for file in files:
r = envoy.run(
'python libraries/gprof2dot.py -f pstats "%s.pstats" | dot -Tpng -o "%s.png"'
% (file, file))
except Exception:
print 'Could not generate image outputs from profiling. Is graphviz installed?'
sys.exit()
def get_gprs_usage(self, target_imsi=None):
"""Get all available GPRS data, or that of a specific IMSI (experimental).
Will return a dict of the form: {
'ipaddr': '192.168.99.1',
'downloaded_bytes': 200,
'uploaded_bytes': 100,
}
Or, if no IMSI is specified, multiple dicts like the one above will be
returned as part of a larger dict, keyed by IMSI.
Args:
target_imsi: the subsciber-of-interest
"""
response = envoy.run('/OpenBTS/OpenBTSCLI -c "gprs list"',
timeout=self.cli_timeout)
if response.status_code != 0:
raise InvalidRequestError('CLI returned with non-zero status: %d' %
response.status_code)
result = {}
for ms_block in response.std_out.split('MS#'):
try:
# Get the IMSI.
match = re.search(r'imsi=[\d]{15}', ms_block)
imsi = 'IMSI%s' % match.group(0).split('=')[1]
# Get the IP.
match = re.search(r'IPs=\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}',
ms_block)
ipaddr = match.group(0).split('=')[1]
# Get the uploaded and downloaded bytes.
match = re.search(r'Bytes:[0-9]+up\/[0-9]+down', ms_block)
count = match.group(0).split(':')[1]
uploaded_bytes = int(count.split('/')[0].strip('up'))
downloaded_bytes = int(count.split('/')[1].strip('down'))
except AttributeError:
# No match found.
continue
except ValueError:
# Casting to int failed.
continue
# See if we already have an entry for the same IMSI -- we sometimes see
# duplicates. If we do have an entry already, sum the byte counts across
# entries.
if imsi in result:
uploaded_bytes += result[imsi]['uploaded_bytes']
downloaded_bytes += result[imsi]['downloaded_bytes']
result[imsi] = {
'ipaddr': ipaddr,
'uploaded_bytes': uploaded_bytes,
'downloaded_bytes': downloaded_bytes,
}
# If, after all that parsing, we still haven't found any matches, return
# None instead of the empty dict.
if result == {}:
return None
# If a specific IMSI was specified, return its data alone if it's in the
# result. If it's not in the parsed result, return None.
if target_imsi and target_imsi not in result.keys():
return None
elif target_imsi:
return result[target_imsi]
# If no IMSI was specified, return all of the parsed data.
return result
def post_archive(post, update_date):
post_id = post['hash']
get = requests.get('%s%s/bookmark/%s' % (es_host, es_index, post_id))
indexed = get.json['exists']
o = urlparse(post['href'])
archive_directory = 'archive/%s/' % (post['hash'])
fetched = os.path.exists(archive_directory)
if indexed:
changed = post['meta'] != get.json['_source']['meta']
else:
changed = False
if changed:
url_changed = post['href'] != get.json['_source']['url']
else:
url_changed = False
#print "url_changed", url_changed
#print "fetched", fetched
#print "changed", changed
#print "indexed", indexed
archive_path = '%s%s%s' % (archive_directory, o.netloc, o.path)
if len(o.query):
archive_path = '%s?%s' % (archive_path, o.query)
if not fetched or url_changed:
print "wget -E -H -k -K -p -P 'archive/%s' '%s'" % (post['hash'], post['href'])
e = envoy.run("wget -E -H -k -K -p -P 'archive/%s' '%s'" % (post['hash'], post['href']))
match = re.search('(%s.*)\' saved' % re.escape(archive_path), e.std_err)
archive_url = 'http://localhost:8080/%s' % match.group(1)
archive_path = '/vagrant/%s' % match.group(1)
changed = True
else:
if indexed:
archive_url = get.json['_source']['archive_url']
archive_path = archive_url.replace('http://localhost:8080/', '/vagrant/')
else:
archive_directory = os.path.dirname(archive_path)
archive_path = '/vagrant/%s/%s' % (archive_directory, [path for path
in os.listdir(archive_directory)
if path.endswith('.html')][0])
archive_url = archive_path.replace('/vagrant/', 'http://localhost:8080/')
#print archive_url
#print archive_path
#sys.exit()
#print "changed", changed
if not indexed or changed:
#sys.exit()
e2 = envoy.run(".venv/bin/python extractor.py %s" % archive_path)
post_text = e2.std_out.replace('\n', ' ')
#html = open(archive_path).read()
#unicode_html = UnicodeDammit(html)
#extractor = Extractor(extractor='ArticleExtractor', html=html)
#post_text = extractor.getText().replace('\n', ' ')
#unicode_text = UnicodeDammit(post_text, unicode_html.original_encoding).unicode_markup
url = 'http://localhost:9200/bookmarks/bookmark/%s/_update' % post['hash']
post_data = {
"title": post['description'],
"url": post['href'],
"text": post_text.replace('"', '\\"'),
"meta": post['meta'],
"posts_update": update_date,
"hash": post['hash'],
"archive_url": archive_url,
}
data = {
"doc": post_data,
"upsert": post_data,
}
#print json.dumps(data).encode('utf-8')
r = requests.post(url, data=json.dumps(data).encode('utf-8'))
print r.status_code
image = Image.open(os.path.join(inpath, filename))
drawing = ImageDraw.Draw(image)
drawing.rectangle([(0, 90), (30, 120)], (255, 255, 255))
drawing.text((0, 90), 's %s' % steering, green)
drawing.text((0, 100), 't %s' % throttle, green)
image.save(os.path.join(tmp_dir, filename))
i += 1
if i % 100 == 0 or i == file_count:
sys.stdout.write('\rwriting telemtry.. %0.1f%%' %
(100. * i / file_count))
sys.stdout.flush()
# Setup path to the images with telemetry.
full_path = os.path.join(tmp_dir, 'frame_*.png')
# Setup output path to video.
output_video_path = os.path.join(out_path, 'telemetry.mp4')
# Run ffmpeg.
command = ("""ffmpeg
-framerate 30/1
-pattern_type glob -i '%s'
-c:v libx264
-r 30
-pix_fmt yuv420p
-y
%s""" % (full_path, output_video_path))
response = envoy.run(command)
print '\nsaving video to %s' % output_video_path
def main(args):
DATABASE = settings.TABULAR_DATABASE
_, ext = os.path.splitext(args.file)
ext = ext.strip('.').lower()
print "Detected extension: " + ext
is_url = args.file.startswith(('http://', 'https://'), )
if args.inspect:
if is_url:
cmd = 'wget "{filename}" -O - | in2csv --no-inference -f {ext} ' \
'| csvstat'.format(filename=args.file, ext=ext)
else:
cmd = 'csvstat "{filename}"'.format(filename=args.file)
result = envoy.run(cmd)
print result.std_out
exit(0)
dbname = args.database
dbport = DATABASE.get("PORT")
dbhost = DATABASE.get('HOST')
if not dbname:
print 'Setting db to default:', DATABASE['NAME']
dbname = DATABASE['NAME']
psql_on_db = "psql -d {dbname} -U {dbuser} -p {dbport} -h {dbhost}" \
"".format(
dbname=dbname,
dbuser=DATABASE['USER'],
dbport=dbport or '5432',
dbhost=dbhost or 'localhost',
)
connection_string = "postgresql://{dbuser}:{dbpass}@" \
"{dbhost}:{dbport}/{dbname}".format(
dbuser=DATABASE['USER'],
dbpass=DATABASE['PASSWORD'],
dbhost=dbhost or 'localhost',
dbport=dbport or '5432',
dbname=dbname
)
print "starting ingestion"
tablename = args.tablename
print "table name:", tablename
if args.erase:
delete_table(psql_on_db, tablename)
exit(0)
delete_table(psql_on_db, tablename)
delimiter_options = "-d '{}'".format(args.delimiter)
if args.delimiter == '\\t':
delimiter_options = '--tabs'
# hack: quotechar is given always escaped (see tab-configuration.json)
if args.quotechar.startswith('\\') and len(args.quotechar) > 1:
args.quotechar = args.quotechar[1:]
separator = iter(frozenset(('"', "'")) - frozenset(args.quotechar)).next()
quotechar_options = "-q {0}{1}{0}".format(separator, args.quotechar)
mime_encoding = args.encoding
if not mime_encoding:
result = envoy.run('file --brief --mime-encoding "{}"'.format(
args.file))
mime_encoding = result.std_out.strip()
print "Using MIME encoding:", mime_encoding
if is_url:
cmd = """wget "{filename}" -O - |
in2csv -f {ext} {delimiter_opts} {quotechar_opts} |
csvsql --no-constraints {delimiter_opts} {quotechar_opts} --insert --db
"{connection_string}" --table "{tablename}" --maxfieldsize 10485760
-e "{mime}" --no-inference -""".format(
filename=args.file,
ext=ext,
connection_string=connection_string,
tablename=tablename,
delimiter_opts=delimiter_options,
quotechar_opts=quotechar_options,
mime=mime_encoding,
)
result = envoy.run(cmd)
if result.status_code:
print_error_result(result, "Error while ingesting data. Aborting.")
exit(1)
else:
# Note: the --maxfieldsize option is set to 10MB
if ext in ("xls", "xlsx"):
cmd = r"""
in2csv -f {ext} {delimiter_opts} {quotechar_opts}
"{filename}" | csvsql --no-constraints {delimiter_opts}
{quotechar_opts} --insert -e "{mime}" --db
"{connection_string}" --table "{tablename}"
--maxfieldsize 10485760 --no-inference -
"""
else:
# Note: the -z option is set to 10MB
cmd = r"""
csvsql --no-constraints {delimiter_opts} {quotechar_opts}
--insert -e "{mime}" --db "{connection_string}"
--table "{tablename}" "{filename}"
--maxfieldsize 10485760 --no-inference
"""
cmd = cmd.format(
mime=mime_encoding,
ext=ext,
filename=args.file,
connection_string=connection_string,
tablename=tablename,
delimiter_opts=delimiter_options,
quotechar_opts=quotechar_options,
)
result = envoy.run(cmd)
if result.status_code:
print_error_result(result, "Error while ingesting data" + cmd)
exit(1)
print "Ingestion completed."
def process_audio(self, file_string):
"""
Processes an uploaded file with `self.file_name`, which
should exist in the temp directory.
"""
timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
# Determine the extension.
file_name, file_extension = os.path.splitext(self.file_name)
# Append the timestamp so we don't clobber similarly named files from the past.
file_name = '%s-%s' % (timestamp, file_name)
file_type, data = file_string.split(';')
prefix, data = data.split(',')
decoded_file = base64.b64decode(data)
if not os.path.exists('/tmp/%s' % app_config.PROJECT_SLUG):
os.makedirs('/tmp/%s' % app_config.PROJECT_SLUG)
with open('/tmp/%s/%s' % (app_config.PROJECT_SLUG, self.file_name),
'wb') as writefile:
writefile.write(decoded_file)
# If mp3, convert to wav for processing in ogg.
if file_extension == ".mp3":
os.system('mpg123 -w "/tmp/%s/%s-temp.wav" "/tmp/%s/%s"' %
(app_config.PROJECT_SLUG, file_name,
app_config.PROJECT_SLUG, self.file_name))
wav_location = "/tmp/%s/%s-temp.wav" % (app_config.PROJECT_SLUG,
file_name)
# If wav, go directly to processing in ogg.
if file_extension == ".wav":
wav_location = '/tmp/%s/%s' % (app_config.PROJECT_SLUG,
self.file_name)
# Encode an OGA.
os.system('oggenc -m 128 -M 128 -o "/tmp/%s/%s.oga" "%s"' %
(app_config.PROJECT_SLUG, file_name, wav_location))
# No matter what, process to 96kb mp3.
os.system('lame -m s -b 128 "%s" "/tmp/%s/%s.mp3"' %
(wav_location, app_config.PROJECT_SLUG, file_name))
# Remove the WAV file now that we've processed it.
os.system('rm -f %s' % wav_location)
# Remove the original file.
os.system('rm -f /tmp/%s/%s' %
(app_config.PROJECT_SLUG, self.file_name))
# If on production/staging, write the file to S3.
if app_config.DEPLOYMENT_TARGET in ['staging', 'production']:
for extension, content_type in [('oga', 'audio/ogg'),
('mp3', 'audio/mpeg')]:
s3_path = '%s/live-data/audio/%s.%s' % (
app_config.PROJECT_SLUG, file_name, extension)
s3 = boto.connect_s3()
for bucket_name in app_config.S3_BUCKETS:
bucket = s3.get_bucket(bucket_name)
k = Key(bucket, s3_path)
k.set_contents_from_filename(
'/tmp/%s/%s.%s' %
(app_config.PROJECT_SLUG, file_name, extension),
headers={
'Content-Type': content_type,
'Cache-Control': 'max-age=5'
})
k.set_acl('public-read')
setattr(
self, 'rendered_%s_path' % extension,
'http://%s.s3.amazonaws.com/%s' %
(app_config.S3_BUCKETS[0], s3_path))
# Remove the temp files.
os.system('rm -f /tmp/%s/%s.%s' %
(app_config.PROJECT_SLUG, file_name, extension))
os.system('rm -f /tmp/%s/%s.%s' %
(app_config.PROJECT_SLUG, file_name, extension))
# If local, write the file to www/live-data/audio.
else:
if not os.path.exists('www/live-data/audio'):
os.makedirs('www/live-data/audio')
envoy.run('mv /tmp/%s/%s.oga www/live-data/audio/' %
(app_config.PROJECT_SLUG, file_name))
envoy.run('mv /tmp/%s/%s.mp3 www/live-data/audio/' %
(app_config.PROJECT_SLUG, file_name))
self.rendered_mp3_path = '/%s/live-data/audio/%s.mp3' % (
app_config.PROJECT_SLUG, file_name)
self.rendered_oga_path = '/%s/live-data/audio/%s.oga' % (
app_config.PROJECT_SLUG, file_name)
# Remove the temp files.
os.system('rm -f /tmp/%s/%s.oga' %
(app_config.PROJECT_SLUG, file_name))
os.system('rm -f /tmp/%s/%s.mp3' %
(app_config.PROJECT_SLUG, file_name))
def adbLog(queue, adb_cmd, timeout):
r = envoy.run(adb_cmd, timeout=timeout)
j = r.std_out
queue.put(j)
def handle(self):
r = envoy.run('task list')
if r.status_code == 0:
print r.std_out
else:
print r.std_err
def main(ctx):
which = "where" if sys.platform == "win32" else "which"
if envoy.run(which + " git").status_code != 0:
click.secho("fatal: git not found in PATH", fg="red")
sys.exit(1)
ctx.obj = Hydrogen()
def optimize(self, ):
"""this does some heavy lifting
unix/mac only feature; sorry.
this function creates an infile and outfile via NamedTemporaryFile
it then pipes the file through lossless compression options
this will replace the self.file object
"""
if self.format_standardized not in ('jpg', 'png', 'gif'):
return
log.debug("optimizing a file. format is: %s" %
self.format_standardized)
FilelikePreference = None
if isinstance(self.file, cStringIO.OutputType):
FilelikePreference = cStringIO.StringIO
else:
FilelikePreference = tempfile.SpooledTemporaryFile
# we need to write the image onto the disk with an infile and outfile
# this does suck.
self.file.seek(0)
fileInput = tempfile.NamedTemporaryFile()
if hasattr(self.file, 'getvalue'):
fileInput.write(self.file.getvalue())
elif hasattr(self.file, 'read'):
fileInput.write(self.file.read())
else:
raise ValueError("not sure what to do")
fileInput.seek(0)
fileOutput = tempfile.NamedTemporaryFile()
_fname_input = fileInput.name
_fname_output = fileOutput.name
# we need this for filesavings
filesize_original = utils.file_size(fileInput)
_optimized = False
if self.format_standardized == 'jpg':
_progressive = '-progressive' if OPTIMIZE_JPEG_PROGRESSIVE else ''
r = envoy.run(
"""jpegtran -copy all -optimize %s -outfile %s %s""" %
(_progressive, _fname_output, _fname_input))
if r.status_code != 127:
_optimized = True
r = envoy.run("""jpegoptim --strip-all -q %s""" %
(_fname_output, ))
if r.status_code != 127:
_optimized = True
elif self.format_standardized == 'gif':
_gifsicle_level = OPTIMIZE_GIFSICLE_LEVEL
r = envoy.run("""gifsicle -O%d %s --output %s""" %
(_gifsicle_level, _fname_input, _fname_output))
if r.status_code != 127:
_optimized = True
elif self.format_standardized == 'png':
if OPTIMIZE_PNGCRUSH_USE:
# envoy.run("""pngcrush -rem alla -reduce -brute -q %s %s""" % (_fname_input, _fname_output))
# envoy.run("""pngcrush -rem alla -reduce -q %s %s""" % (_fname_input, _fname_output))
r = envoy.run(
"""pngcrush -rem alla -nofilecheck -bail -blacken -reduce -cc %s %s"""
% (_fname_input, _fname_output))
if r.status_code != 127:
_fname_input = _fname_output
_optimized = True
if OPTIMIZE_OPTIPNG_USE:
_optipng_level = OPTIMIZE_OPTIPNG_LEVEL
# note that we do `--out OUTPUT --(stop) INPUT
r = envoy.run("""optipng -i0 -o%d -out %s -- %s""" %
(_optipng_level, _fname_output, _fname_input))
if r.status_code != 127:
_fname_input = _fname_output
_optimized = True
if OPTIMIZE_ADVPNG_USE:
_advpng_level = OPTIMIZE_ADVPNG_LEVEL
# note that we do `--out OUTPUT --(stop) INPUT
r = envoy.run("""advpng -%d -z %s""" %
(_advpng_level, _fname_output))
if r.status_code != 127:
_fname_input = _fname_output
_optimized = True
fileOutput.seek(0)
newFile = FilelikePreference()
newFile.write(fileOutput.read())
newFile.seek(0)
self.file = newFile
self.is_optimized = True
# so how much did we save?
filesize_optimized = utils.file_size(newFile)
optimization_savings = filesize_original - filesize_optimized
log.debug("optimization_savings = %s" % optimization_savings)
self.optimization_savings = optimization_savings
import tarfile
import os
import envoy
import urllib.request
import shutil
url = "https://drive.google.com/file/d/1nhBeJoVyAs1q9Ld53b_kotx6DuS2sfYl/view?usp=sharing"
path_save = "data\\save\\20news-bydate.tar.gz"
if not os.path.exists(path_save):
urllib.request.urlretrieve(url, path_save)
# file = "data\\20news-bydate.tar.gz"
# path_save = "data\\data_20news"
if (path_save.endswith("tar.gz")):
envoy.run("tar -xzf %s -C %s" % (path_save, "data"))
shutil.unpack_archive(path_save, 'data')
def youtube_thumbnail(youtube_link):
exe = "youtube-dl --get-thumbnail "+ youtube_link
output = envoy.run(exe.encode('ascii'))
output_str = output.std_out
return output_str
def fit(self, X, y): # X = TAGsets, y = labels
""" Trains classifier
input: list of tags [list] and labels [list] for ALL training tagsets
output: trained model
"""
start = time.time()
if not self.probability: # probability has to do with whether or not it is multilabel
X, y = self._filter_multilabels(X, y)
if self.use_temp_files and not self.iterative:
modelfileobj = tempfile.NamedTemporaryFile('w', delete=False)
self.vw_modelfile = modelfileobj.name
modelfileobj.close()
else:
#self.vw_modelfile = 'trained_model-%s.vw' % self.suffix
if not (self.iterative and self.trained):
safe_unlink(self.vw_modelfile)
else:
logging.info("Using old vw_modelfile: %s", self.vw_modelfile)
logging.info('Started hybrid model, vw_modelfile: %s',
self.vw_modelfile)
self.vw_args_ = self.vw_args
if not (self.iterative and self.trained):
self.indexed_labels = {}
self.reverse_labels = {}
self.all_labels = set()
self.label_counter = 1
else:
# already have a trained model
self.vw_args_ += ' -i {}'.format(self.vw_modelfile)
# add labels in y to all_labels
for labels in y:
if isinstance(labels, list):
for l in labels:
self.all_labels.add(l)
else:
self.all_labels.add(labels)
for label in sorted(list(self.all_labels)):
if label not in self.indexed_labels:
self.indexed_labels[label] = self.label_counter
self.reverse_labels[self.label_counter] = label
self.label_counter += 1
################################################
## Create VW arg string ########################
if self.probability:
self.vw_args_ += ' --csoaa {}'.format(len(self.all_labels))
else:
self.vw_args_ += ' --probabilities'
self.loss_function = 'logistic'
self.vw_args_ += ' --loss_function={}'.format(self.loss_function)
if self.iterative:
self.vw_args_ += ' --oaa 80'
else:
self.vw_args_ += ' --oaa {}'.format(len(self.all_labels))
if self.iterative:
self.vw_args_ += ' --save_resume'
self.vw_args_ += ' --kill_cache --cache_file a.cache'
####################################################
train_set = list(zip(X, y))
if self.use_temp_files:
f = tempfile.NamedTemporaryFile('w', delete=False)
else:
with open('./label_table-%s.yaml' % self.suffix, 'w') as f:
yaml.dump(self.reverse_labels, f)
f = open('./fit_input-%s.txt' % self.suffix, 'w')
for tag, labels in train_set:
if isinstance(labels, str):
labels = [labels]
input_string = ''
if self.probability:
for label, number in self.indexed_labels.items():
if label in labels:
input_string += '{}:0.0 '.format(number)
else:
input_string += '{}:1.0 '.format(number)
else:
input_string += '{} '.format(self.indexed_labels[labels[0]])
f.write('{}| {}\n'.format(input_string, ' '.join(tag)))
f.close()
# write all tag/label combos into a file f ^^^
######## Call VW ML alg ##################################
command = '{vw_binary} {vw_input} {vw_args} -f {vw_modelfile}'.format(
vw_binary=self.vw_binary, vw_input=f.name,
vw_args=self.vw_args_, vw_modelfile=self.vw_modelfile)
#logging.info('vw input written to %s, starting training', f.name)
#logging.info('vw command: %s', command)
vw_start = time.time()
c = envoy.run(command)
##########################################################
### Print info about VW run ##############################
#logging.info("vw took %f secs." % (time.time() - vw_start))
print("vw took this many seconds: ", (time.time() - vw_start))
if c.status_code:
logging.error(
'something happened to vw, code: %d, out: %s, err: %s',
c.status_code, c.std_out, c.std_err)
raise IOError('Something happened to vw')
else:
logging.info(
'vw ran sucessfully. out: %s, err: %s',
c.std_out, c.std_err)
if self.use_temp_files: # WILL USUALLY BE FALSE
safe_unlink(f.name)
self.trained = True # once the fit function has been run, model has been trained!
logging.info("Training took %f secs." % (time.time() - start))
def compile(filename, print_path=False):
realfile = filename
dirpath = dirname(filename) + '/'
outpath = join(dirpath.replace('/coffee/', '/js/', 1),
basename(realfile)[:-7] + '.js')
with open(filename) as coffeescript:
m = []
for l in coffeescript:
import_by, ln = import_by_line(dirpath, l)
if import_by:
for name in import_by:
add_import(name, realfile)
if exists(name):
with open(name) as js:
m.append('#--- %s\n' % name)
for line in js:
m.append(' ' * ln + line)
m.append('#--- %s\n' % name)
else:
print '\t#include %s NOT EXIST' % name
filename = None
else:
m.append(l)
m = ''.join(m)
filename = join(TMP_DIR, md5(m).hexdigest() + '.coffee')
print realfile
if not exists(filename):
with open(filename, 'w') as output:
output.write(m)
if filename.endswith('.coffee'):
jspath = filename[:-7]
else:
jspath = filename
jspath += '.js'
if print_path:
print realfile
if outpath not in HG_IGNORE:
write_ignore(outpath)
if not exists(jspath):
cmd = 'coffee -c %s' % (filename)
r = envoy.run(cmd)
err = r.std_err
if err:
print ''
print realfile
print err.split('\n')[0].split(',', 1)[-1].strip()
print ''
if exists(jspath):
makedirs(outpath)
if exists(outpath):
try:
rmtree(outpath)
except OSError:
pass
makedirs(dirname(outpath))
copy(jspath, outpath)
for i in IMPORT_BY[realfile]:
compile(i, False)
def predict_proba(self, X): # X = tagsets
"""Calculates probability of any of the labels that the model has been
trained on belonging to each tagset in X
input: list of tagsets
output: probabilities for each label and each tagset
"""
start = time.time()
if not self.trained:
raise ValueError("Need to train the classifier first")
#tags = self._get_tags(X) (X = tags)
if self.use_temp_files:
f = tempfile.NamedTemporaryFile('w', delete=False)
outfobj = tempfile.NamedTemporaryFile('w', delete=False)
outf = outfobj.name
outfobj.close()
else:
f = open('./pred_input-%s.txt' % self.suffix, 'w')
outf = './pred_output-%s.txt' % self.suffix
if self.probability:
for tag in X:
f.write('{} | {}\n'.format(
' '.join([str(x) for x in self.reverse_labels.keys()]),
' '.join(tag)))
else:
for tag in X:
f.write('| {}\n'.format(' '.join(tag)))
f.close()
logging.info('vw input written to %s, starting testing', f.name)
args = f.name
args += ' -r %s' % outf
command = '{vw_binary} {args} -t -i {vw_modelfile}'.format(
vw_binary=self.vw_binary, args=args,
vw_modelfile=self.vw_modelfile)
logging.info('vw command: %s', command)
vw_start = time.time()
c = envoy.run(command)
logging.info("vw took %f secs." % (time.time() - vw_start))
if c.status_code:
logging.error(
'something happened to vw, code: %d, out: %s, err: %s',
c.status_code, c.std_out, c.std_err)
raise IOError('Something happened to vw')
else:
logging.info(
'vw ran sucessfully. out: %s, err: %s',
c.std_out, c.std_err)
all_probas = []
with open(outf, 'r') as f:
for line in f:
probas = {}
for word in line.split(' '):
tag, p = word.split(':')
probas[tag] = float(p)
all_probas.append(probas)
if self.use_temp_files:
safe_unlink(f.name)
safe_unlink(outf)
if not self.iterative:
safe_unlink(self.vw_modelfile)
logging.info("Testing took %f secs." % (time.time() - start))
return all_probas
def remove_port(self, port):
if self.is_port_exist(port) is True:
ovs_cmd = envoy.run('%s del-port %s' % (util.OVS_VSCTL_CMD, port))
if ovs_cmd.status_code != 0:
raise net_agentd_exception.NetAgentException(
"remove port failed:\n%s" % ovs_cmd.std_err)
def setup(file, nodeploy):
"""Set up a new Web Framework Django server instance"""
print "\nSetting up the Splunk Web Framework..."
try:
splunk_home = os.environ.get("SPLUNK_HOME", "")
splunk_5 = None
splunkd_scheme = None
splunkweb_scheme = None
splunkd_port = None
splunkweb_port = None
splunkd_host = None
splunkweb_host = None
splunkweb_mount = None
splunkdj_mount = None
splunkdj_appserver_port = None
splunkdj_proxy_port = None
splunkdj_proxy_path = None
while True:
version_info = envoy.run(
[['%s/bin/splunk' % splunk_home, 'version']])
version = version_info.std_out.strip()
if not (version.startswith("Splunk 5")
or version.startswith("Splunk 6")
or version.startswith("Splunk 201")):
os.remove(path.join(MAIN_DIR, ".splunkhome"))
print "The version must be >= 'Splunk 5.0', found '%s' in '%s'. Run 'splunkdj setup' and try again." % (
version, splunk_home)
sys.exit(1)
if splunk_5 == None:
splunk_5 = version.startswith("Splunk 5")
# Get Python, Node and Splunk paths
splunk_path = path.join(splunk_home, "bin",
"splunk" + (".exe" if is_win32 else ""))
python_path = path.join(splunk_home, "bin",
"python" + (".exe" if is_win32 else ""))
node_path = path.join(splunk_home, "bin",
"node" + (".exe" if is_win32 else ""))
python_exists = path.exists(python_path.strip())
node_exists = path.exists(node_path)
# Ensure Python and Node exist
if not python_exists:
print "No Python interpreter, exiting..."
sys.exit(1)
if not node_exists:
print "No Node.js interpreter, exiting..."
sys.exit(1)
# Get Various information from Splunk
if not splunkd_port:
splunkd_port = get_conf_value("web", "settings",
"mgmtHostPort").split(":")[1]
if not splunkweb_port:
splunkweb_port = get_conf_value("web", "settings", "httpport")
if not splunkweb_mount:
splunkweb_mount = get_conf_value("web", "settings",
"root_endpoint")
if not splunkd_scheme:
is_splunkd_ssl = normalizeBoolean(
get_conf_value("server", "sslConfig", "enableSplunkdSSL"))
splunkd_scheme = "https" if is_splunkd_ssl else "http"
if not splunkweb_scheme:
is_splunkweb_ssl = normalizeBoolean(
get_conf_value("web", "settings", "enableSplunkWebSSL"))
splunkweb_scheme = "https" if is_splunkweb_ssl else "http"
splunkd_scheme = splunkd_scheme or "https"
splunkd_host = splunkd_host or "localhost"
splunkweb_scheme = splunkweb_scheme or "http"
splunkweb_host = splunkweb_host or "localhost"
splunkweb_mount = splunkweb_mount or ""
splunkdj_mount = splunkdj_mount or "dj"
splunkdj_appserver_port = splunkdj_appserver_port or SPLUNKDJ_APPSERVER_DEFAULT_PORT
splunkdj_proxy_port = splunkdj_proxy_port or SPLUNKDJ_PROXY_DEFAULT_PORT
splunkdj_proxy_path = splunkdj_proxy_path or SPLUNKDJ_PROXY_DEFAULT_PATH
print "\nThe Splunk Web Framework will use the following values:"
print " - Splunkd scheme: %s" % splunkd_scheme
print " - Splunkd host: %s" % splunkd_host
print " - Splunkd port: %s" % splunkd_port
print " - Splunk Web scheme: %s" % splunkweb_scheme
print " - Splunk Web host: %s" % splunkweb_host
print " - Splunk Web port: %s" % splunkweb_port
print " - Splunk Web root endpoint: %s" % splunkweb_mount
print " - Web Framework Django appserver port: %s" % splunkdj_appserver_port
print " - Web Framework proxy port: %s" % splunkdj_proxy_port
print " - Web Framework proxy path: %s" % splunkdj_proxy_path
print " - Web Framework mount: %s" % splunkdj_mount
print " - Splunk installation (SPLUNK_HOME): %s" % splunk_home
print " - Splunk 5: %s" % splunk_5
if confirm(
"\nAre these values correct ('y' to accept, 'n' to edit)",
default=True):
break
splunkd_scheme = raw_input("Splunkd scheme [%s]: " %
(splunkd_scheme)) or splunkd_scheme
splunkd_host = raw_input("Splunkd host [%s]: " %
(splunkd_host)) or splunkd_host
splunkd_port = raw_input("Splunkd port [%s]: " %
(splunkd_port)) or splunkd_port
splunkweb_scheme = raw_input(
"Splunk Web scheme [%s]: " %
(splunkweb_scheme)) or splunkweb_scheme
splunkweb_host = raw_input("Splunk Web host [%s]: " %
(splunkweb_host)) or splunkweb_host
splunkweb_port = raw_input("Splunk Web port [%s]: " %
(splunkweb_port)) or splunkweb_port
splunkweb_mount = raw_input("Splunk Web mount [%s]: " %
(splunkweb_mount)) or splunkweb_mount
# Get information about Web Framework ports
splunkdj_appserver_port = raw_input(
"Web Framework Django appserver port [%s]: " %
(splunkdj_appserver_port)) or splunkdj_appserver_port
while is_port_open("localhost", splunkdj_appserver_port):
if confirm(
"Web Framework Django appserver port '%s' is taken. Would you like to change it"
% splunkdj_appserver_port,
default=True):
splunkdj_appserver_port = raw_input(
"Web Framework appserver port [%s]: " %
(splunkdj_appserver_port)) or splunkdj_appserver_port
else:
sys.exit(1)
splunkdj_proxy_port = raw_input(
"Web Framework proxy port [%s]: " %
(splunkdj_proxy_port)) or splunkdj_proxy_port
while is_port_open("localhost", splunkdj_proxy_port):
if confirm(
"Web Framework proxy port '%s' is taken. Would you like to change it"
% splunkdj_proxy_port,
default=True):
splunkdj_proxy_port = raw_input(
"Web Framework proxy port [%s]: " %
(splunkdj_proxy_port)) or splunkdj_proxy_port
else:
sys.exit(1)
splunkdj_proxy_path = raw_input(
"Web Framework proxy path [%s]: " %
splunkdj_proxy_path) or splunkdj_proxy_path
splunkdj_mount = raw_input("Web Framework mount [%s]: " %
splunkdj_mount) or splunkdj_mount
splunk_home = raw_input(
"Splunk installation (SPLUNK_HOME) [%s]: " %
splunk_home) or splunk_home
splunk_home = path.expanduser(splunk_home)
splunk_5 = normalizeBoolean(
raw_input("Splunk 5 [%s]: " % splunk_5) or splunk_5)
# Write out SPLUNK_HOME
dot_splunkhome = open(path.join(MAIN_DIR, '.splunkhome'), 'w')
dot_splunkhome.write(splunk_home)
dot_splunkhome.flush()
# Serialize configuration
create_config_file(config_file_path=path.join(MAIN_DIR, file),
splunkd_scheme=splunkd_scheme,
splunk_home=splunk_home,
splunkd_host=splunkd_host,
splunkd_port=int(splunkd_port),
splunkweb_scheme=splunkweb_scheme,
splunkweb_host=splunkweb_host,
splunkweb_port=int(splunkweb_port),
splunkweb_mount=splunkweb_mount,
x_frame_options_sameorigin=normalizeBoolean(
get_conf_value("web", "settings",
"x_frame_options_sameorigin")),
mount=splunkdj_mount,
raw_mount=splunkdj_mount,
splunkdj_port=int(splunkdj_appserver_port),
proxy_port=int(splunkdj_proxy_port),
proxy_path=splunkdj_proxy_path,
debug=False,
quickstart=False,
splunk_5=splunk_5)
if not nodeploy:
print "\nInstalling default apps..."
while True:
username = raw_input("Splunk username: "******"Splunk password: "******"",
force=True,
file=DEFAULT_SPLUNKDJ_CONFIG_FILE,
username=username,
password=password,
)
for app in SPLUNKDJ_DEFAULT_APPS:
# Only deploy the default apps if they actually exist
app_path = path.join(MAIN_DIR, "server", "apps", app)
if not path.exists(app_path):
continue
args.appname = app
deploy(args.appname, args.force, args.file, args.username,
args.password)
break
except KeyboardInterrupt:
print
sys.exit(0)
print "\nThe Splunk Web Framework setup is complete -- enter 'splunkdj run' to start."
def check_domain_available(domain):
r = envoy.run('whois %s'%domain)
if 'No match for' in r.std_out:
return True
return False
def main():
args = vars(parser.parse_args())
# Configure action
if args["action"] == "configure":
cfg_parser = ConfigParser.SafeConfigParser()
cfg_parser.add_section("aws")
cfg_parser.set("aws", "access_key", raw_input("AWS Access Key: "))
cfg_parser.set("aws", "secret_key", raw_input("AWS Secret Key: "))
cfg_parser.set("aws", "s3_bucket", raw_input("S3 Bucket Name: "))
cfg_parser.set("aws", "folder", raw_input("folder: "))
cfg_parser.add_section("mongodb")
cfg_parser.set("mongodb", "host", raw_input("MongoDB Host: "))
cfg_parser.set("mongodb", "username", raw_input("MongoDB Username: "******"mongodb", "password", raw_input("MongoDB Password: "******"~/.mo2s3.cfg"), "w"))
print "Config written in %s" % os.path.expanduser("~/.mo2s3.cfg")
sys.exit(0)
if not args["access_key"] or not args["secret_key"]:
print "S3 credentials not set, run 'mo2s3 configure' or specify --access-key/--secret-key, 'mo2s3 -h' to show the help"
sys.exit(0)
# Amazon S3 connection
conn = boto.connect_s3(args["access_key"], args["secret_key"])
bucket = conn.get_bucket(args["bucket"])
folder = args["folder"]
now = datetime.now().strftime("%Y%m%d%H%M%S")
print "S3 Bucket: " + args["bucket"]
print "S3 Folder: " + args["folder"]
def key_for_filename(filename):
if folder:
return folder + '/' + filename
else:
return filename
# Backup action
if args["action"] == "backup":
print "MongoDB: " + args["host"]
# Run mongodump
dump_directory = "mongodump_" + now
mongo_cmd = "mongodump" + make_mongo_params(
args) + " -o " + dump_directory
if not args["db"]:
mongo_cmd += " --oplog"
print mongo_cmd
mongodump = envoy.run(mongo_cmd)
if mongodump.status_code != 0:
print mongodump.std_err
print mongodump.std_out
# Create dump archive
tar_filename = "mongodump"
if args["db"]:
tar_filename += "_" + args["db"]
if args["collection"]:
tar_filename += "_" + args["collection"]
tar_filename += "_" + now + ".tgz"
tar = envoy.run("tar czf " + tar_filename + " " + dump_directory)
if tar.status_code != 0:
print tar.std_err
print tar.std_out
# Upload to S3
k = Key(bucket)
k.key = key_for_filename(tar_filename)
k.set_contents_from_filename(tar_filename)
print tar_filename + " uploaded"
os.remove(tar_filename)
try:
shutil.rmtree(dump_directory)
except Exception:
print "shutil.rmtree({dump_directory}) got an error (does {dump_directory} really exist?)"\
.format(dump_directory=dump_directory)
# List action
elif args["action"] == "list":
# List bucket files
for key in bucket.list(prefix=folder):
print key.name
# print folder
# if folder:
# for key in bucket.list(prefix=folder):
# print key.name
# else:
# for key in bucket.get_all_keys():
# print key.name
# Drop action
elif args["action"] == "drop":
# Delete every file in the bucket
for key in bucket.list(folder):
print "deleting " + key.name
key.delete()
# Delete action
elif args["action"] == "delete":
# Delete the backup on S3
if not args["filename"]:
print "No filename specified (--filename), 'mo2s3 -h' to show the help"
sys.exit(0)
k = Key(bucket)
k.key = key_for_filename(args["filename"])
print "deleting " + args["filename"]
k.delete()
# Restore action
elif args["action"] == "restore":
print "MongoDB: " + args["host"]
if not args["filename"]:
print "No filename specified (--filename), 'mo2s3 -h' to show the help"
sys.exit(0)
# Download backup file from S3
k = Key(bucket)
k.key = key_for_filename(args["filename"])
print "restoring " + args["filename"]
k.get_contents_to_filename(args["filename"])
dump_date = args["filename"][-18:-4]
tar = envoy.run("tar xvzf " + args["filename"])
if tar.status_code != 0:
print tar.std_err
print tar.std_out
# Run mongorestore
restore_cmd = "mongorestore" + make_mongo_params(
args) + " mongodump_" + dump_date
if args["db"]:
restore_cmd += "/" + args["db"]
if args["collection"]:
restore_cmd += "/" + args["collection"]
else:
restore_cmd += " --oplogReplay"
print restore_cmd
def _templates_js():
r = envoy.run('node_modules/.bin/jst --template underscore jst')
return r.std_out, 200, {'Content-Type': 'application/javascript'}
def mov_main_profile(in_fn, out_fn):
r = envoy.run(
'ffmpeg -i %s -profile main -acodec copy -vcodec libx264 %s' %
(in_fn, out_fn))
print r.std_err
