def genData():
services = []
for i in range(0, util.randomInt(2)):
services.append({
"date":
util.randomDate(afterDate="03/01/2020 8:00 AM",
toDate="05/15/2020 05:00 PM"),
"name":
util.randomString(util.randomInt(2)),
"code":
util.randomInt(4),
"description":
util.randomString(util.randomInt(3)),
"cost":
round(util.randomMoney(min=100.00, max=900.00), 2)
})
return {
"patient": {
"patientId": util.randomInt(12),
"name": util.generateName(),
"address": util.randomStreetAddress(),
"city": util.randomCity(),
"state": util.randomState(),
"zip": util.randomZipCode()
},
"fileId": util.randomInt(6),
"created": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"services": services
}
def newToken(self, username, wait=False):
self.eternal()
token = randomString(16)
while (self.getUsername(token, wait=True)):
token = randomString(16)
self.c.execute('insert into session (token, username) values (%s, %s)',
(token, username))
self.conn.commit()
ret = token
if not (wait):
self.disconnect()
return ret
def api_image_run(sid: uuid.UUID, port: int):
user: User = xtoken_user(AuthAPI.getXToken())
tag = randomString(15)
db: wrappers.Collection = mongo.db.images
image: Image = deserialize_json(
Image, db.find_one({
"uid": user.uuid,
"uuid": str(sid)
}))
try:
container_id = DockerImageAPI.run(image.tag, "", port)
container_uuid = str(uuid.uuid4())
container = Container(user.uuid, tag, "start", str(sid), port,
container_id, container_uuid)
db: wrappers.Collection = mongo.db.containers
db.insert_one(container.__dict__)
DockerContainerAPI.start(container)
docker_daemon.notify(container_id)
return json_result(0, "image run")
except:
return json_result(-1, "image not found")
def backup(freenasHostname, outputPath):
#pull backup configuration from pfsense
remotePath = 'root@' + freenasHostname + ':/data/freenas-v1.db'
outputDirectory = outputPath + '/' + freenasHostname + '/'
tmpDir = mkdtemp()
downloadedConfig = tmpDir + '/freenas-' + date.today().isoformat() + '-' + util.randomString(5) + '.db'
call(['rsync', remotePath, downloadedConfig])
#checks for changes to the configuration and moves the file if there are.
if path.isdir(outputDirectory):
#this returns the newest config file so it can be compared to the one just downloaded
latestConfigFile = check_output(['ls', '-t', outputDirectory], universal_newlines=True).split('\n').pop(0)
else:
#if there's no directory, make it.
latestConfigFile = None
call(['mkdir', '-p', outputDirectory])
#if there's nothing in the directory, backup the file
if latestConfigFile is None:
call(['cp', downloadedConfig, outputDirectory])
#if there are files in the directory, we need to check and see if there have been any changes to the config
#by taking a hash of it and comparing it to the current file
elif not util.areFilesEqual(outputDirectory + latestConfigFile, downloadedConfig):
call(['cp', downloadedConfig, outputDirectory])
#remove the tmp directory
call(['rm', '-rf', tmpDir])
def createRandomObject(fluiddb):
"""
Creates an object with a random about tag using /about.
"""
about = 'benchmark test object ' + randomString(25)
fluiddb.about.post(about)
return about
def backup(esxiHostname, outputPath):
#pull backup configuration from esxi
tmpDir = mkdtemp()
outputDirectory = outputPath + '/' + esxiHostname + '/'
downloadedConfig = tmpDir + '/esxi-' + date.today().isoformat() + '-' + util.randomString(5) + '.tgz'
backupCommands = 'vim-cmd hostsvc/firmware/backup_config'
#make the backup on esxi
backupOutput = check_output(['ssh', 'root@' + esxiHostname, backupCommands])
#ESXi makes the backup file available at a URL
#these operations strip the command's output down to just the URL
backupOutput = backupOutput.decode('UTF-8').rstrip('\n')
backupOutput = backupOutput.split(' ').pop().replace('*', esxiHostname)
#download the file to the tmp directory
call(['wget', '-O', downloadedConfig, backupOutput, '--no-check-certificate'])
#checks for changes to the configuration and moves the file if there are.
#get the current backups sorted by date
if path.isdir(outputDirectory):
latestConfigFile = check_output(['ls', '-t', outputDirectory], universal_newlines=True).split('\n').pop(0)
else:
call(['mkdir', '-p', outputDirectory])
latestConfigFile = None
#if there's nothing in the directory, backup the file
if latestConfigFile is None:
call(['cp', downloadedConfig, outputDirectory])
#if there are files in the directory, we need to check and see if there have been any changes to the config
#by taking a hash of it and comparing it to the current file
elif not util.areFilesEqual(outputDirectory + latestConfigFile, downloadedConfig):
call(['cp', downloadedConfig, outputDirectory])
call(['rm', '-rf', tmpDir])
def buildExperiment(net,wa,wb,wc,visibility=False):
simName = util.randomString(20);
net.add_to_nengo()
numsensors=4 # number of agents sensors (if changed, need to read correct values in the Controller)
simulator = oneAgent(net,agentName='a',numSensors=numsensors,maxDistance=120, frictionDistance=0,nameSpace=simName,visible=visibility);
vivae = simulator.getControls();
# Input->LI
controller = net.add(EventGenerator('Agent controller',numsensors*2+1,2,1)); # event generator
input = net.make_input('bias1',[1]) # should result in 0, 0, .33
net.connect(input, controller.getTermination('params'))
net.connect(simulator.getAgent('a').getOrigin(),controller.getTermination('inputs')) # connect agent with controller
# LII->LIII->Output
input2 = net.make_input('bias2',[1])
summer = net.add(Summer('Summator'));
net.connect(input2,summer.getTermination('inputBias'));
net.connect(controller.getOrigin('outputs'),summer.getTermination('inputData'))
net.connect(summer.getOrigin('outputs'),simulator.getAgent('a').getTermination())
saver = net.add(SpeedSaver('avg'))
net.connect(simulator.getAgent('a').getOrigin(),saver.getTermination('data'));
#saver = net.add(DataSaver('saver')) # save data ??
return simulator
def createRandomObject(fluiddb):
"""
Creates an object with a random about tag using /objects.
"""
about = 'benchmark test object ' + randomString(25)
response = fluiddb.objects.post(about)
return response.value['id']
def createRandomNamespace(fluiddb, path):
"""
Creates a namespace with a random name.
"""
name = randomString(25)
description = 'Namepsace ' + name
fluiddb.namespaces[path].post(name, description)
return path.rstrip('/') + '/' + name
def createRandomTag(fluiddb, path):
"""
Creates a new tag with a random name.
"""
name = randomString(25)
description = 'Tag ' + name
fluiddb.tags[path].post(name, description, True)
return path.rstrip('/') + '/' + name
def segment_frame_restoration():
filename = './' + randomString() + '.mp4'
filename2 = './' + randomString() + '.mp4'
v = open(filename, "wb")
segment = base64.b64decode(request.form['b64'])
v.write(segment)
#v.seek(0)
v.close()
width, height = get_video_size(filename)
print(width, height)
reader = start_ffmpeg_reader(filename)
writer = start_ffmpeg_writer(filename2, width * 2, height * 2)
time_elapsed = 0
while True:
start = timer()
in_frame = read_frame(reader, width, height)
if in_frame is None:
break
frame_sr = sr_genarator(model, in_frame)
write_frame(writer, frame_sr)
end = timer()
elapsed = end - start
time_elapsed += (elapsed)
print('Time per frame: {}'.format(elapsed))
print('LR frame shape: {}'.format(in_frame.shape))
print('SR frame shape: {}'.format(frame_sr.shape))
# payload = payloader_pre(in_frame)
# # Making POST request
# r = requests.post('http://10.0.0.118:8501/v1/models/upscale2x:predict', json=payload)
# print('Status code: {}'.format(r.status_code))
# if(r.status_code==200):
# frame_sr = payloader_pos(r)
print('Time elapsed: {}'.format(time_elapsed))
reader.wait()
writer.stdin.close()
writer.wait()
os.remove(filename)
return return_seg(filename2)
def tokenNew(self, username, length=64):
while (True):
try:
token = randomString(length)
us = self.tokenLookup(token)
except DBException:
break
self.c.execute('insert into session (username, token) value (%s, %s)',
(username, token))
return token
def doSignup():
id = randomString(16)
usr = request.form.get("usr")
pwd = request.form.get("pwd") + '#$%^&*$@' + usr
eml = request.form.get("eml")
pwd = md5(pwd.encode())
acc = Account()
ret = acc.add((id, usr, pwd.digest(), eml))
if ret > 0:
return redirect("/authen/signin")
return render_template("authen/signup.html", err="Username Existed")
def add():
cart = Cart()
id = request.cookies.get('cart')
if id == None:
id = randomString(16)
response = make_response(redirect('/cart'))
response.set_cookie('cart', id, max_age= 3600 * 24 * 30)
else:
response = redirect('/cart')
productid = request.form.get('productid')
quantity = request.form.get('quantity')
cart.add((id, productid, quantity))
return response
def getId(cls, name: str, sourceId: int) -> int:
assert (isinstance(sourceId, int))
es = Author.select({'name': name}, f'''
case when exists (
select 1 from author_source s
where s.authorId = authorId and s.sourceId = {sourceId}
) then 1 else 0 end desc''')
if len(es) > 1:
util.logMessage(f'many authors: name={name} sourceId={sourceId}')
if len(es) >= 1:
return es[0].id
if len(es) > 0:
raise Exception('FIXME')
e = Author.new()
e.name = name
e.urlId = util.randomString(8, charset=util.urlIdCharset)
e.insert()
return Author.getId(name, sourceId)
def doSignin():
usr = request.form.get('usr')
pwd = request.form.get('pwd') + '#$%^&*$@' + usr
rem = request.form.get('rem')
pwd = md5(pwd.encode())
acc = Account()
v = acc.getAccount((usr, pwd.digest()))
if v:
res = make_response(redirect("/authen"))
token = randomString(32)
ses = Session()
ret = ses.add((token, v['id']))
if rem == "1":
res.set_cookie("session", token, max_age=30 * 3600 * 24)
else:
res.set_cookie("session", token)
return res
else:
return render_template('authen/signin.html', err="Log In Failed")
def getPostUpdatedOrPublished(self, post: Any) -> int:
# old style xen foro
messageMeta = post.find_all('div', {'class': 'messageMeta'})
if len(messageMeta) == 1:
dt = messageMeta[0].find_all('span', {'class': 'DateTime'})
ts = None
if len(dt) == 1:
dt = dt[0]
ts = dt.get('title')
else:
dt = messageMeta[0].find_all('abbr', {'class': 'DateTime'})
if len(dt) != 1:
raise Exception(
'error: unable to find message meta datetime')
dt = dt[0]
ts = dt.get_text()
tsp = dateutil.parser.parse(ts)
uts = util.dtToUnix(tsp)
return uts
if len(messageMeta) > 1:
raise Exception('error: unable to find message meta')
# new xen foro style
lastEdit = post.find('div', {'class': 'message-lastEdit'})
if lastEdit is not None:
t = lastEdit.find('time')
return int(t.get('data-time'))
postPublish = post.find('div', {'class': 'message-attribution-main'})
if postPublish is not None:
t = postPublish.find('time')
return int(t.get('data-time'))
postPublish = post.find('header', {'class': 'message-attribution'})
if postPublish is not None:
t = postPublish.find('time')
return int(t.get('data-time'))
edumpContent(str(post), 'xen_post' + util.randomString())
raise Exception('unable to find post update or publish ts')
def urlNew(self, url, short=None, username=None, length=3):
if short is None:
while (True):
try:
short = randomString(length)
_ = self.urlLookup(short)
except:
break
else:
try:
_ = self.urlLookup(short)
except:
pass
else:
raise DBException('Short URL exists')
self.c.execute('insert into map (id, url) value (%s, %s)',
(short, url))
if not (username is None):
self.c.execute('insert into own (username, id) value (%s, %s)',
(username, short))
return short
def setRandomTagValueString(fluiddb, objectId, path):
"""
Updates a tag on the given object using a random string as value.
"""
value = randomString(100)
return fluiddb.objects[objectId][path].put(value)
def __init__(self) -> None:
super().__init__()
self._cachedFandoms: Optional[List[Fandom]] = None
self.urlId = util.randomString(8, charset=util.urlIdCharset)
self.importStatus = ImportStatus.pending
pCross = 0.9
popsize = 3
maxgen = 3
if config == 2:
t = 2
pMut = 0.3
pCross = 0.9
popsize = 50
maxgen = 100
if config == 3:
t = 2
pMut = 0.4
pCross = 0.9
popsize = 20
maxgen = 100
simName = util.randomString(20)
net = nef.Network('Vivae - Turning controller for agent')
net.add_to_nengo()
ea = EA(maxgen, popsize, minw, maxw, genomeLen)
ea.setProbabilities(pMut, pCross)
ea.initPop()
simulator = buildExperiment(net, wa, wb, wc, visibility=False)
vivae = simulator.getControls()
expNo = round(1000000 * random.random(), 0)
# generate some number for text file data
print expNo
f = open('data/tmp/ea_%d.txt' % expNo, 'w')
sx = Saver('ea_%d_agents.txt' % expNo)
# saves best agent from actual generation during the evolution into a file
# evolution insert here
def extractContent(self, fic: Fic, html: str) -> str:
from bs4 import BeautifulSoup
contentId = util.randomString(8)
while html.find(contentId) >= 0:
contentId = util.randomString(len(contentId) + 1)
soup = BeautifulSoup(f'<div id="{contentId}">{html}</div>', 'html5lib')
# SB
for spoiler in soup.find_all('div', {'class': 'bbCodeSpoiler'}):
button = spoiler.find('button')
title = spoiler.find('span',
{'class': 'bbCodeSpoiler-button-title'})
if title is not None and button is not None:
t = soup.new_tag('span')
t.append(title.get_text())
button.insert_after(t)
if button is not None:
button.extract()
for spoiler in soup.find_all('div',
{'class': 'bbCodeSpoiler-content'}):
spoiler.attrs['class'] = 'spoiler'
# QQ
for spoiler in soup.find_all('div',
{'class': 'bbCodeSpoilerContainer'}):
spoiler.attrs.pop('class')
spoiler.name = 'span'
for spoiler in soup.find_all('div', {'class': 'bbCodeSpoilerText'}):
spoiler.attrs['class'] = 'spoiler'
# for the proxy js based img tags, fiddle with their attributes so the
# html cleanup code gets the proxy url out of .data-url and the original
# upstream url from .src (or the proxy url if we don't have a real
# upstream)
for img in soup.find_all('img'):
# proxy img tags have data-src but no actual src
if 'data-src' not in img.attrs:
continue
if 'src' in img.attrs:
continue
src = img.attrs['data-src']
if not src.startswith('http'):
src = self.baseUrl + src
altSrc = None
if 'data-url' in img.attrs:
altSrc = img.attrs['data-url']
img.attrs['data-url'] = src
img.attrs['src'] = src
if altSrc:
img.attrs['src'] = altSrc
# general 'click to expand' nonsense
for div in soup.find_all('div', {'class': 'quoteExpand'}):
if div.get_text().strip() in {
'Click to expand...', 'Click to expand…'
}:
div.extract()
# CloudFlare protected "emails"
for e in soup.find_all('a', {'class': '__cf_email__'}):
if 'data-cfemail' not in e.attrs:
continue
t = e.get_text()
if not t.startswith('[email') or not t.endswith('protected]'):
continue
cfemail = e.attrs['data-cfemail']
email = util.decodeCloudFlareEmail(cfemail)
util.logMessage(f'decoding email|{cfemail}|{email}')
e.name = 'span'
e.attrs.clear()
e.string = email
content = soup.find('div', {'id': contentId})
content = content.contents
if isinstance(content, list):
content = content[0]
return str(content)
def setRandomTagValueString(fluiddb, about, path):
"""
Updates a tag on the object with the given about tag.
"""
value = randomString(100)
return fluiddb.about[about][path].put(value)
def UpdateTagWithRandomDescription(fluiddb, path):
"""
Updates a tag with a random value.
"""
description = randomString(50)
return fluiddb.tags[path].put(description)
def nextGeneration(job):
"""
iterate the optimization to the next generation:
1. Sorts current generation by cost
2. Crosses the best two
3. Randomly mutates the rest
4. Creates new random candidates to keep the generation size constant
5. Removes "old" jobs because the time to search/index takes longer and longer
and in this case, we don't really need to keep all of them around
"""
startTime = time.time()
project = job._project
gNum = project.document.generation.n
# find all jobs in this generation
lGeneration = project.find_jobs(filter={'master': False}, doc_filter={'generation': gNum})
# create a list and sort
costList = [(j.get_id(), j.document.cost) for j in lGeneration]
sortList = np.array(costList, dtype=[('id', '|S32'), ('cost', int)])
sortList.sort(order='cost')
print("generation: {}, cost: {}".format(gNum, sortList[0][1]))
# now we go through and create the next generation
for i, (lID, lCost) in enumerate(sortList):
# get the job
lJob = project.open_job(id=lID.decode('UTF-8'))
# cross the best two
if i == 0:
codeA = lJob.sp.code
jobB = project.open_job(id=sortList[1][0].decode('UTF-8'))
codeB = jobB.sp.code
newA, newB = util._mate(codeA, codeB)
# create new jobs
statepoint = dict(length=len(job.sp.goal),
goal=job.sp.goal,
code=newA,
seed=lJob.sp.seed,
master=False)
project.open_job(statepoint).init()
lA = project.open_job(statepoint)
lA.document.generation = gNum + 1
statepoint = dict(length=len(job.sp.goal),
goal=job.sp.goal,
code=newB,
seed=lJob.sp.seed,
master=False)
project.open_job(statepoint).init()
lB = project.open_job(statepoint)
lB.document.generation = gNum + 1
# mutate remaining sans last 4 (effectively drops the worst 4)
if i < (len(sortList) - 4):
newCode = util._mutate(lJob.sp.code, 0.5)
statepoint = dict(length=len(job.sp.goal),
goal=job.sp.goal,
code=newCode,
seed=lJob.sp.seed,
master=False)
project.open_job(statepoint).init()
lJob = project.open_job(statepoint)
lJob.document.generation = gNum + 1
# find how many more statepoints are required to keep the generation size constant
lGeneration = project.find_jobs(filter={'master': False}, doc_filter={'generation': gNum+1})
nRemain = len(sortList) - len(lGeneration)
while nRemain > 0:
for i in range(nRemain):
# generate a new random statepoint
code = util.randomString(len(job.sp.goal))
statepoint = dict(length=len(job.sp.goal),
goal=job.sp.goal,
code=code,
seed=job.sp.seed,
master=False)
project.open_job(statepoint).init()
lJob = project.open_job(statepoint)
lJob.document.generation = gNum + 1
lGeneration = project.find_jobs(filter={'master': False}, doc_filter={'generation': gNum+1})
nRemain = len(sortList) - len(lGeneration)
project.document.generation.n = gNum + 1
# remove "old" jobs to keep the optimization fast
oldJobs = project.find_jobs(filter={'master': False}, doc_filter={'generation': {'$lt': gNum-2}})
for j in oldJobs:
j.remove()
stopTime = time.time()
benchmark_doc.time[str(gNum)] = stopTime
benchmark_doc.njobs[str(gNum)] = len(project.find_jobs())
from flask import *
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from util import randomString, print_to_console
from datetime import datetime, timedelta
import os
from os.path import join, dirname, realpath
app = Flask(__name__)
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///app.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SECRET_KEY'] = randomString()
app.config["UPLOAD_FOLDER"] = join(dirname(realpath(__file__)),
r"static\images\uploads")
db = SQLAlchemy(app)
migrate = Migrate(app, db)
from models import *
@app.route('/')
def index():
try:
user_id = session['user_id']
user = User.query.filter_by(id=user_id).first()
events = Event.query.all()
return redirect(url_for('upcomming'))
def docker_build():
"""
GET
:param uid: user uuid
POST
:param tag: docker tag
:parma dockfile: dockerfile uuid
:param rootpass: root password for ssh
:param sshport: ssh port forwarding
build Dockerfile
"""
if request.method != "POST":
return json_result(-1, "POST only")
uid = session.get("uuid")
username = session.get("username")
tag = request.form["tag"]
dockfile = request.form["dockfile"]
rootpass = request.form["rootpass"]
sshport = int(request.form["sshport"])
fn = "upload/{}/{}/Dockerfile".format(username, dockfile)
with open(fn, "r") as f:
df = f.read()
name = tag.split(":")[0]
ver = "latest"
if len(tag.split(":")) == 1:
ver = tag.split(":")[1]
tag = randomString(20 - len(name)) + name + ":" + ver
image_uuid = str(uuid.uuid4())
container_uuid = str(uuid.uuid4())
image = Image(uid, "", tag, "installing", sshport, "", image_uuid)
db: wrappers.Collection = mongo.db.images
db.insert_one(image.__dict__)
# search Dockerfile
df: wrappers.Collection = mongo.db.dockerfile
result: Dockerfile = deserialize_json(Dockerfile,
df.find_one({"uuid": dockfile}))
if result == None:
return json_result(-1, "Dockerfile is not exist")
try:
# image build
image.status = "build"
db.update({"uuid": image_uuid}, image.__dict__)
result, imgs = DockerImageAPI.build(result.path, rootpass, tag)
image.short_id = imgs[0]["Id"].split(":")[1]
print(result)
image.status = "done"
db.update({"uuid": image_uuid}, image.__dict__)
except:
image.status = "fail"
db.update({"uuid": image_uuid}, image.__dict__)
return json_result(-1, "Dockerfile::Image::build fail")
# container start
container_id = DockerImageAPI.run(tag, "",
sshport) # image.run(tag, port=sshport)
container = Container(uid, tag, "start", image_uuid, sshport, container_id,
container_uuid)
container.start(container_id)
db: wrappers.Collection = mongo.db.containers
db.insert_one(container.__dict__)
docker_daemon.notify(container_id)
result_stream = []
for item in result:
try:
result_stream += [item["stream"]]
except:
continue
return json_result(0, "".join(result_stream))
class DataSaver(nef.SimpleNode):
def tick(self):
i=file('data/sensoryData.csv','a+') # data from robot sensors
o=file('data/actuatorData.csv','a+') # data comming from regulator to robot actuators
sensorydata = simulator.getAgent('a').getOrigin().getValues().getValues(); # columns: time, [distance sensors from the left, friction sensors from the left]
actuatordata = controller.getOrigin('outputs').getValues().getValues(); # columns: time, left_wheel, right_wheel
i.write('%1.3f,%s\n'%(self.t,list(sensorydata)))
o.write('%1.3f,%s\n'%(self.t,list(actuatordata)))
i.close()
o.close()
#################################################################################
simName = util.randomString(20);
net=nef.Network('Vivae - Turning controller for agent')
net.add_to_nengo()
numsensors=4 # number of agents sensors (if changed, need to read correct values in the Controller)
simulator = oneAgent(net,agentName='a',numSensors=numsensors,maxDistance=120, frictionDistance=0,nameSpace=simName);
vivae = simulator.getControls();
#controller = net.add(Controller('Agent controller',2*numsensors+1,2,0)); # build controller
controller = net.add(TurnGenerator('Agent controller',2*numsensors+1,2,3)); # build explicit controller
input=net.make_input('input',[20, 0.0, 0.33])
net.connect(input,controller.getTermination('params'))
net.connect(simulator.getAgent('a').getOrigin(),controller.getTermination('inputs')) # connect agent with controller
net.connect(controller.getOrigin('outputs'), simulator.getAgent('a').getTermination()) # connect controller to the agent
def updateNamespaceWithRandomDescription(fluiddb, path):
"""
Updates a namespace with a random description.
"""
description = randomString(50)
return fluiddb.namespaces[path].put(description)