def __init__(self, mem_size: int = 10000):
"""
The following is an abstraction for a bank of values
such as valA, which will be used during each cycle.
It's set up as an object to avoid circular import.
"""
self.ValBank = ValBank()
"""
The following are functional units like memory,
registers, or flags
"""
self.Memory = Memory(mem_size)
self.RegisterBank = RegisterBank()
self.ZF = CCFlag("ZF") # zero flag
self.OF = CCFlag("OF") # overflow flag
self.SF = CCFlag("SF") # sign flag
self.ErrorFlag = StateFlag("Error Flag", error_lib)
self.StateFlag = StateFlag("State Flag", state_lib)
self.ALU = ALU(self.ValBank, self.StateFlag, self.ErrorFlag, self.SF, self.OF, self.ZF)
"""
The following are functional abstractions of operations
that the processor performs
"""
self.Fetcher = Fetcher(self.ValBank, self.RegisterBank, self.Memory, self.StateFlag, self.ErrorFlag)
self.Decoder = Decoder(self.ValBank, self.RegisterBank, self.Memory)
self.Executor = Executor(self.ValBank, self.ALU, self.OF, self.ZF, self.SF)
self.Memorizer = Memorizer(self.ValBank, self.Memory)
self.RegWriter = RegWriter(self.RegisterBank, self.ValBank)
self.PCUpdater = PCUpdater(self.RegisterBank, self.ValBank)
def __init__(self):
Executor.__init__(self)
chromedriver = "/home/andrew/Projects/03_Pinscraper/webdrivers/chromedriver"
self._driver = webdriver.Chrome(executable_path=chromedriver)
#todo declare Curator
self._Curator = Curator("pinterest")
#todo data and solvers
#data should be a class every appended object
#parent access, make a new class that inherits data
self._data = [self._driver, self._Curator]
def checkScript():
global sessionId
executor = Executor()
while True:
time.sleep(executor.getCheckTime())
executor.execScript()
if sessionId == None:
break
def main():
Process = Executor()
Process.add(FirstClass())
SecondProcess = Executor()
SecondProcess.add(SecondClass())
Process.add(SecondProcess)
Process()
def __init__(self, ip, port=9090, max_users=20):
self.console = False
self.HOST = ip
self.PORT = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.bind((self.HOST, self.PORT))
self.sock.listen(max_users)
self.blacklist = []
self.users = []
self.ex = Executor(self)
self.th = threading.Thread(target=self.acceptor)
self.th.start()
class Processor:
"""
This represents the processor itself. Most work and operations
will be outsourced to other objects referenced by the processor.
The processor holds the values of valP, valE, valA, valB, valC,
and rA, rB.
"""
def __init__(self, mem_size: int = 10000):
"""
The following is an abstraction for a bank of values
such as valA, which will be used during each cycle.
It's set up as an object to avoid circular import.
"""
self.ValBank = ValBank()
"""
The following are functional units like memory,
registers, or flags
"""
self.Memory = Memory(mem_size)
self.RegisterBank = RegisterBank()
self.ZF = CCFlag("ZF") # zero flag
self.OF = CCFlag("OF") # overflow flag
self.SF = CCFlag("SF") # sign flag
self.ErrorFlag = StateFlag("Error Flag", error_lib)
self.StateFlag = StateFlag("State Flag", state_lib)
self.ALU = ALU(self.ValBank, self.StateFlag, self.ErrorFlag, self.SF, self.OF, self.ZF)
"""
The following are functional abstractions of operations
that the processor performs
"""
self.Fetcher = Fetcher(self.ValBank, self.RegisterBank, self.Memory, self.StateFlag, self.ErrorFlag)
self.Decoder = Decoder(self.ValBank, self.RegisterBank, self.Memory)
self.Executor = Executor(self.ValBank, self.ALU, self.OF, self.ZF, self.SF)
self.Memorizer = Memorizer(self.ValBank, self.Memory)
self.RegWriter = RegWriter(self.RegisterBank, self.ValBank)
self.PCUpdater = PCUpdater(self.RegisterBank, self.ValBank)
def run(self):
"""
This is the only necessary public method for the processor.
Currently the way to operate this is by manually loading values
into the memory using the place_instruction method, then calling
the 'run' method for the processor. Afterwards calling print
on the memory object will reveal the finish state of the processor.
"""
while self.StateFlag.State == 0:
self.Fetcher.fetch()
self.Decoder.decode()
self.Executor.execute()
self.Memorizer.memory_write()
self.RegWriter.write_back()
self.PCUpdater.update_pc()
def evalSolutions(self, solutions):
solVecs = []
for solution in solutions:
solVecs.append(solution.getSolVec())
executor = Executor()
objs = executor.evalFns(solVecs, self.objs)
constr = executor.evalFns(solVecs, self.constrs)
idx = 0
for solution in solutions:
solution.setFitness(objs[idx])
solution.setInfact(constr[idx])
idx += 1
return solutions
def __init__(self):
self.objs = []
self.constrs = []
self.Executor = Executor()
self.objMap = {}
self.ctrMap = {}
pass
def main(argv) -> None:
executor: Executor = Executor(Path.cwd()).exec(argv)
flexio_flow: FlexioFlow = FlexioFlow(executor.config())
if executor.config().options.debug:
flexio_flow.process()
else:
try:
flexio_flow.process()
except KeyboardInterrupt:
PrintColor.log(Fg.FOCUS.value + "\n\n" +
'### Flex bye bye budy ! ###' + "\n")
except (FileNotFoundError, FileExistsError, ImportError,
AttributeError, ValueError, KeyError, NotImplementedError,
GitMergeConflictError, NotADirectoryError, TypeError,
IndexError, GithubRequestApiError, ConnectionError,
FlexioRequestApiError) as error:
sys.stderr.write("""
{red}#######################################
# OUPS !!!
# {type}:{error}
#######################################{reset}
""".format(red=Fg.FAIL.value,
type=error.__class__.__name__,
error=error,
reset=Fg.RESET.value))
sys.stderr.write("Command terminated with wrong status code: 1" +
"\n")
sys.exit(1)
sys.exit(0)
def __init__(self):
self.executor = Executor()
self.original_file = None
self.original_file_name = None
self.mutation_types = None
self.program = None
self.mutator = None
def __set_exec_threads(self): #sets a threads for each tape exec_threads = [] for i in range(len(self.mt.tapes)): exec_threads.append(Executor(self.mt.tapes[i], self.transition.post_conditions[i], self.transition.movements[i])) exec_threads[i].start() return exec_threads
def run_experiment(config, setting, num_repetitions):
#set policies
policies = setting.set_policies(config.K)
num_policies = len(policies)
#set regrets
regrets = setting.set_regrets()
num_regrets = len(regrets)
#execute policies and compute the regrets
exe = Executor()
curr_regret = np.zeros((num_policies, num_regrets, config.N))
for pol_idx in range(num_policies):
# execute each policy num_repetition times
for ii in range(num_repetitions):
pulls = exe.run_policy(config, policies[pol_idx])
# regret update
for reg_idx in range(num_regrets):
regr = regrets[reg_idx].compute_regret(pulls, config)
curr_regret[pol_idx, reg_idx, :] = curr_regret[
pol_idx, reg_idx, :] + np.true_divide(
regr, num_repetitions)
# before starting a new repetition generate a new configuration and reset the state of the policy
config.gen_stochastic_processes()
policies[pol_idx].reset_state()
#plot the regrets
titles = []
labels = []
styles = []
filename = config.name + '_'
for policy in policies:
labels = labels + [policy.name]
styles = styles + [policy.style]
filename = filename + policy.name + '-'
filename = filename[:-1] + '_'
for regret in regrets:
titles = titles + [regret.description]
filename = filename + regret.name + '-'
plt = Plotter()
plt.plot_regret(curr_regret, titles, labels, styles, filename[:-1])
class Requst_processor(object):
"""
This class have infinite cycle for read, parse and run command, send reply.
stmp - send mail
imap - read mail
executor - make changes in sheet
"""
def __init__(self, gmail, password, sheets):
self.smtp = SMTP(gmail, password)
self.imap = IMAP(gmail, password)
self.executor = Executor(sheets)
def run(self):
period = 0
while True:
self.imap.chec_mail()
# print(self.imap.get_comand())
for comand in self.imap.get_comand():
try:
print('-', comand)
self.executor.execute(comand) # return status and comand
#self.executor.transfer() # ?? try &&
except Exception as e:
str_exeption = str(type(e)) + str(e.args)
write_log(log=self.reply_on_comand(
comand['address'], comand['command'], str_exeption))
else:
write_log(log=self.reply_on_comand(
comand['address'], comand['command'], 'SUCCESS'))
period += 1
time.sleep(25)
def reply_on_comand(self, address, command, result):
self.smtp.send(address, result)
if result == 'SUCCESS':
print(result)
else:
print('ERROR ', result)
result = result
return '/' + address + '/' + command + '/' + result
def main():
"""
All the experiments on all the configurations and all the policies have all the same true mean.
The generation of the stochastic processes from this true means is different for each experiment.
"""
#log config
logging.basicConfig(filename=st.LOG_FILENAME,level=logging.INFO)
exe = Executor()
for config in st.configs:
str_info= '\n Executing experiments of configuration ' + config.name + ' with N=' + str(st.N) + ' and num_rep=' + str(st.num_repetitions) + ' ...'
logging.info(str_info)
print(str_info)
start_time = time.clock()
exe.run_configuration(config, st.policies, st.num_repetitions, st.N)
exp_time = time.clock() - start_time
str_info = 'TIME TO PERFORM EXPERIMENTS OF ' + config.name + 'CONFIGURATION: ' + str(exp_time)
logging.info(str_info)
print(str_info)
def _getBoards(self, driver):
boards = driver.find_elements_by_css_selector('div.AggregatedCloseupCard.Module.inDidIt')
for board in boards:
board_link = board.find_element_by_css_selector("a[href*='pin']")
driver.get(board_link.get_attribute("href"))
#return Navigator(url= board_link.get_attribute("href"))
board_container = driver.find_element_by_css_selector("div.feedContainer.gatorFeed")
board_summarys = board_container.find_elements_by_class_name("boardSummary")
print(board_summarys)
#todo test collection with Executors
Navigators = Executor()
for summary in board_summarys:
attr = summary.find_element_by_css_selector('a').get_attribute('href')
print("myattr: ", attr)
#link = "www.pinteret.com" + summary.get_attribute("href")
#print(link)
Navigators.append(Navigator(url=attr))
Navigators.append(PinCollector())
print("REturning", Navigators)
return Navigators
def main(args):
#Init settings
Settings.init()
Settings.enableDebug = args.debug
print(args.debug)
print(Settings.enableDebug)
executor = Executor()
validationDataPath = os.path.abspath("ValidationData")
#Load validation data
with open(validationDataPath + '/validator.json') as json_file:
validationData = json.load(json_file)
validationData = validationData['exams']
#load labels
label_file = executor.input.modelPath + '/yolo-obj.names'
labels = open(label_file).read().strip().split('\n')
#init result rate which means % success rate of each label
resultRate = {}
for label in labels:
resultRate[label] = 0
#star validation
for item in validationData:
#execute detection
executeResult = executor.execute(validationDataPath + '/' +
item['image'])
#validate and get result
validationResult = validateResult(item['data'], executeResult, labels)
#add validation result to final result
for label in labels:
resultRate[label] = resultRate[label] + validationResult[label]
showResult(resultRate, validationData)
return
def main():
# start a new node
rospy.init_node("executor")
exe = Executor(
file_path=
'/home/chen/ws_chen/src/hilsys/sim_sys/data/fwx_planned_trajectory.txt'
)
# read the joint trajectory from txt file
exe.read_trajectory()
# initiate the pose with the first joint trajectory point
'''
get_state_service = rospy.ServiceProxy('/gazebo/get_model_state', GetModelState)
model = GetModelStateRequest()
model.model_name = 'mm'
robot_state = get_state_service(model)
set_state_service = rospy.ServiceProxy('/gazebo/set_model_state', SetModelState)
model = SetModelStateRequest()
desired_state=ModelState()
desired_state.model_name='fwx'
desired_state.pose.position.x = exe._base_pose_list[0].x
desired_state.pose.position.y = exe._base_pose_list[0].y
desired_state.pose.orientation=euler_to_quaternion(0,0,math.pi/2-exe._base_pose_list[0].theta)
set_state_service(desired_state)
'''
exe.init_pose()
rospy.loginfo('please press enter to send to trajectory goal')
raw_input()
rospy.loginfo('sending joint action goal')
# go to target pose
# exe.go_to_target_pose([0.315593518306, -0.0583244396675, 0.316494316795, -0.0258618760329, 0.313625004658, 0.104796584025, 0.310698981177])
# go to the last pose
# exe.go_to_last_pose()
# go to other joint trajectory points one by one
#exe.send_trajectory_one_by_one(0.01)
# go to other joint trajectory points
exe.send_trajectory()
#print(ob)
'''
def main():
#set configuration
N = 2**17
num_repetitions = 5
config = MUUD_Configuration(N)
#set possible values of a parameter
values = np.arange(0.1, 1, 0.1)
#set policy
policies = []
policy_names = []
for value in values:
policy = UCB2_Policy(alpha=value)
policy.name = policy.name + str(value)
policies = policies + [policy]
policy_names = policy_names + [policy.name]
#execute experiments
exe = Executor()
print('Executing experiments for tunining...')
start_time = time.clock()
exe.run_configuration(config, policies, num_repetitions)
exp_time = time.clock() - start_time
print('TIME TO PERFORM EXPERIMENTS: ', exp_time)
#printing the results
print('Printing tuning results...')
start_time = time.clock()
plot_configuration(config.name, policy_names)
exp_time = time.clock() - start_time
print('TIME TO PRINT RESULTS: ', exp_time)
def pinHolders(self, driver):
print("Getting PinHolders")
Navigators = Executor()
pinHolders = driver.find_elements_by_css_selector("div.pinHolder")
print("PinSaver count: ", PinSaver._ids)
for pinHolder in pinHolders[:10]:
#refine this to test if the pinboard?
link = pinHolder.find_element_by_css_selector('a').get_attribute('href')
#store pinboard in database?
#todo SUPER IMPORTANT CHECK LINK IN THE DATABASE to avoid infinite loop or have navigator hold urls.
if not self.exists(link):
Navigators.append(Navigator(url=link))
Navigators.append(PinSaver())
Navigators.append(Scrollers())
#Navigators.append(BoardCollector())
#Navigators.append(PinCollector())
else:
print("url exists in Curator")
return Navigators
def __init__(self, schedule_parser, report_directory, schedule_name,
report_sql_file, cs_id, tr_id, rq_param):
Executor.__init__(self, schedule_parser, report_directory,
schedule_name, report_sql_file, cs_id, tr_id,
rq_param)
def __init__(self, workloads_dict):
Executor.__init__(self, workloads_dict)
self.AllProcess = []
self.TARGET_CPU_USAGE = 30.0
#!/usr/local/bin/python
from Model import Model
from Executor import Executor
if __name__ == '__main__':
model = Model()
model.assemble_graph()
silence_step = 0
skip_step = 20
exe = Executor(model, silence_step=silence_step, skip_step=skip_step)
exe.train_and_dev()
exe.restore_and_test()
if __name__ == '__main__':
seed = 12345
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
n_gpu = torch.cuda.device_count()
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
if n_gpu > 0:
torch.cuda.manual_seed(seed)
silence_step = 0 # 从哪步开始可以进行evaluate
skip_step = 20 # 每隔多少步做evaluate
config = DicToObj(**config_model)
formatter = '%(asctime)s %(levelname)s %(message)s'
config = generate_name_path(config, path_parser)
train_logger = Logger(filename=config.saved_path + '/train.log',
fmt=formatter).logger
tb_logger = Tensorboard_Logger(config.saved_path)
exe = Executor(model_name="stocknet",
config=config,
silence_step=silence_step,
skip_step=skip_step,
train_logger=train_logger,
tb_logger=tb_logger)
exe.apply(xavier_init)
exe.train_and_dev(do_continue=False)
exe.restore_and_test()
class FuzzerClientProtocol(amp.AMP):
def __init__(self):
self.executor = Executor()
self.original_file = None
self.original_file_name = None
self.mutation_types = None
self.program = None
self.mutator = None
def connectionMade(self):
setupDeferreds = [self.getOriginalFile(), self.getProgram(), self.getMutationTypes()]
defer.gatherResults(setupDeferreds).addCallback(self.finishSetup).addErrback(stop)
def finishSetup(self, ign):
# we have all the pieces the mutator needs
self.mutator = Mutator(
self.original_file, self.mutation_types, self.original_file_name, self.factory.tmp_directory)
# enter continuous loop
self.lc = LoopingCall(self.getNextMutation)
self.lc.start(0)
def getNextMutation(self):
''' Ask the server for the next mutation '''
return (self.callRemote(commands.GetNextMutation) # return deferred
.addCallback(self.executeNextMutation)
.addErrback(stop))
def executeNextMutation(self, mutation):
if mutation['stop']:
stop("Server said to stop")
return False
if mutation['pause']:
print 'Sever said to pause - sleeping for 60 seconds'
sleep(60)
return True
# create the mutated file
new_file_name = self.mutator.createMutatedFile(mutation['offset'], mutation['mutation_index'])
# execute it
print '(%d,%d)' % (mutation['offset'], mutation['mutation_index']),
output = self.executor.execute(self.program, new_file_name)
if output:
print 'Got output, Offset = %d, Mutation_Index = %d, File = %s' % (mutation['offset'], mutation['mutation_index'], new_file_name)
self.callRemote( commands.LogResults, mutation_index=mutation['mutation_index'], offset=mutation['offset'], output=output, filename=new_file_name )
# copy the file - it caused a crash
copy("%s"%new_file_name, "%s"%join(self.factory.save_directory, split(new_file_name)[-1]))
# remove the file
remove("%s"%new_file_name)
@defer.inlineCallbacks
def getOriginalFile(self):
''' get original file and file name'''
response = yield self.callRemote(commands.GetOriginalFile)
print '[*] Got original_file :', response['original_file_name']
self.original_file = response['original_file']
self.original_file_name = response['original_file_name']
@defer.inlineCallbacks
def getMutationTypes(self):
''' get list of mutations that will be used '''
response = yield self.callRemote(commands.GetMutationTypes)
print '[*] Got mutation types'
self.mutation_types = response['mutation_types']
@defer.inlineCallbacks
def getProgram(self):
''' get target program that will be executed '''
response = yield self.callRemote(commands.GetProgram)
print '[*] Got program :', response['program']
self.program = response['program']
def __init__(self, schedule_parser, report_directory, schedule_name, report_sql_file, cs_id, tr_id, rq_param):
Executor.__init__(
self, schedule_parser, report_directory, schedule_name, report_sql_file, cs_id, tr_id, rq_param
)
self.AllProcess = []
self.should_stop = False
def main(args):
executor = Executor()
executor.execute(args.file)
return
if len(sys.argv) > 1 and sys.argv[1] == "prod":
#for closed form of communication, 4th argument to specify if it is a closed system or not
if len(sys.argv) == 5 and sys.argv[3] == "closed":
global closed, passphrases
closed = True
passphrases[sys.argv[2]] = sys.argv[4]
global producer, Topic
producer = True
Topic = sys.argv[2]
print "Topic of production: %s " % Topic
subscribers[Topic] = set()
rnd = Random.new().read
print "Generating rsa keys...."
rsa_key = RSA.generate(1024, rnd)
server_public_key = RSA.importKey(server_pub_key)
v_addr = get_virtual_addr()
executor = Executor(v_addr, peers, peer_map, subscribers, passphrases,
server_public_key)
R = Register(('172.17.0.2', 5000))
R.register()
print "Node virtual address: %s" % v_addr
thread.start_new_thread(send_thread, (executor, ))
thread.start_new_thread(recv_thread, (executor, ))
thread.start_new_thread(connect_new_peer, (None, ))
while True:
time.sleep(10)
def __init__(self, url=""):
Executor.__init__(self)
self.seturl(url)
def create_app(cfgfile=None, instance_path=None):
app = Flask(__name__,
instance_path=instance_path,
instance_relative_config=bool(instance_path))
# see if a config file exists and load it
def try_config(filename):
""" if filename exists, load it. return true if load successful """
# test file, allow a 'config' subdir
if not os.path.isfile(filename):
filename = os.path.join('config', filename)
print("Testing for config file", filename)
if os.path.isfile(filename):
print("Loading configuration from", filename)
if filename.endswith('.py'):
app.config.from_pyfile(filename)
elif filename.endswith('.json'):
app.config.from_json(filename)
else:
print("Unknown suffix on file f{filename}, assume pyfile")
app.config.from_pyfile(filename)
return True
return False
if cfgfile:
if not try_config(cfgfile):
raise FileNotFoundError(cfgfile)
else:
testfiles = ('config.' + system() + '.py',
'config.' + hostname() + '.py', 'config.default.py')
for fname in testfiles:
if try_config(fname):
break
app.config.setdefault('DATAPATH', 'data')
app.config.setdefault('LOGFILE', 'logs/CCDDroneGUI.log')
if not app.secret_key:
app.secret_key = 'dev'
# set up logging
logformat = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
loglevel = app.config.get('LOGLEVEL', logging.WARNING)
if app.debug:
loglevel = logging.DEBUG
logfile = app.config.get('LOGFILE')
logging.basicConfig(format=logformat, level=loglevel, filename=logfile)
if logfile:
file_handler = RotatingFileHandler(logfile,
maxBytes=1024 * 1024 * 100,
backupCount=20)
file_handler.setLevel(loglevel)
formatter = logging.Formatter(logformat)
file_handler.setFormatter(formatter)
app.logger.addHandler(file_handler)
# turn down werkzeug logs
#logging.getLogger('werkzeug').setLevel(logging.WARNING)
# set up handlers
BasicAuth(app)
Bootstrap(app)
ImageDB.ImageDB(app=app)
# executor runs the actual programs
app.executor = Executor(app.config)
atexit.register(app.executor.abort)
def getdb():
return app.extensions['ImageDB']
def _getcache(key):
return getdb().getcache(key)
def _setcache(key, val):
getdb().setcache(key, val)
app.add_template_global(hostname)
app.add_template_global(system)
@app.template_global()
def dtype(val):
return type(val).__name__
@app.route('/')
def index():
return render_template("index.html")
@app.route('/api/status')
def status():
return json.jsonify(app.executor.getstatus())
@app.errorhandler(RuntimeError)
@app.errorhandler(FileNotFoundError)
def runtime_error(err):
flash(f"ERROR: {str(err)}", 'danger')
return redirect(url_for('index'))
@app.route('/expose', methods=('GET', 'POST'))
def expose():
cachekey = 'last_exposure_settings_' + system()
# get the last values we used
lastvals = _getcache(cachekey)
if lastvals is None:
lastvals = {'metadata': {}}
lastvals['metadata']['SYSTEM'] = system()
form = ExposeForm(request.form, data=lastvals)
if request.method == "POST" and form.validate():
_setcache(cachekey, form.data)
app.executor.savemetadata(form.metadata.data)
app.executor.ExposeLoop(form.nexposures.data, form.filename.data,
form.exposure.data)
flash('Exposure started', 'success')
return redirect(url_for('index'))
# always default to 1 exposure for new starts
form.nexposures.process_data(1)
return render_template("expose.html", form=form)
@app.route('/startup', methods=('POST', ))
def startup():
app.executor.StartupAndErase()
flash('StartupAndErase started', 'success')
return redirect(url_for('index'))
@app.route('/erase', methods=('POST', ))
def erase():
app.executor.PerformEraseProcedure()
flash('Erase procedure started', 'success')
return redirect(url_for('index'))
@app.route('/togglebias/<value>', methods=('POST', ))
def togglebias(value):
app.executor.ToggleBias(value)
flash(f'ToggleBias {value} executed')
return redirect(url_for('index'))
@app.route('/editconfig', methods=('GET', 'POST'))
def editconfig():
if request.method == 'POST':
app.executor.saveconfig(request.form['config'], 'apply'
in request.form)
flash("Configuration saved", "success")
return redirect(url_for('index'))
config = app.executor.readconfig()
if config is None:
flash(f"Unable to load config file {app.executor.outputConfig}",
'danger')
return render_template(
"editconfig.html",
droneconfig=config if config is not None else "")
@app.route('/abort', methods=('POST', ))
def abortproc():
app.executor.abort()
flash("Abort submitted", 'warning')
return redirect(url_for('index'))
@app.route('/endexposeloop', methods=('POST', ))
def endexposeloop():
app.executor.endexposureloop()
flash("Exposures will end after current one", 'success')
return redirect(url_for('index'))
####### database browser endpoints ###########
@app.route('/show/<filename>')
def showfile(filename):
info = getdb().find_one({'filename': filename})
if not info:
abort(404, f"No registered file with name '{filename}'")
return render_template('showfile.html', fileinfo=info)
@app.route('/api/getimg/<filename>')
def getimg(filename):
datapath = app.config.get('DATAPATH')
filepath = os.path.join(datapath, filename)
if not os.path.isfile(filepath):
abort(404, f"Raw fits file '{filename}' not present")
with tempfile.NamedTemporaryFile(suffix='.png') as tmpfile:
tmpname = tmpfile.name
subprocess.run(
['fits2bitmap', filepath, '-o', tmpname, '--percent', '98'])
return send_file(tmpname)
return abort(500, "something went wrong sending file...")
def datatableentry(item, colnames):
""" Format a mongodb result to an object to put in a DataTable """
res = {name: item.get(name, None) for name in colnames}
filename = res.get('filename')
if filename:
link = f"<a href=\"{ url_for('showfile', filename=filename) }\">"
link += f"{filename}</a>"
res['filename'] = link
return res
@app.route('/api/DataTable', methods=('GET', 'POST'))
def datatable():
""" Process a jquery DataTable request """
req = request.json
# construct a mongo filter object from the request
query = {}
sort = {}
#search for individual column filters
columns = req.get('columns', [])
colnames = [col['name'] for col in columns]
for col in columns:
val = col.get('search', {}).get('value', None)
if val:
query[col['name']] = {'$regex': val}
#test the global filter
globalsearch = req.get('search', {}).get('value', None)
if globalsearch:
query['$or'] = [{
name: {
'$regex': globalsearch
}
} for name in colnames]
#determine ordering
sort = []
for order in req.get('order', []):
sort.append((colnames[order['column']],
-1 if order['dir'] == 'desc' else 1))
if not sort:
sort = (('EXPSTART', -1), )
#return only data for requested columns
projection = {name: True for name in colnames}
projection['_id'] = False
#now query the DB
total = getdb().count()
match = getdb().count(query)
data = []
if match > 0:
cursor = getdb().find(
query,
projection,
sort=sort,
skip=req.get('start', 0),
limit=req.get('length', 0),
)
data = [datatableentry(item, colnames) for item in cursor]
return json.jsonify({
'draw': req['draw'],
'recordsTotal': total,
'recordsFiltered': match,
'data': data,
})
print(req)
# need to construct a mongo filter object from the request
return abort(500)
return json.jsonify(req)
@app.route('/listdata')
def listdata():
columns = ('EXPSTART', 'RUNTYPE', 'NOTES', 'filename')
# find columns that have limited choices
selectOptions = {}
reqmd = getdb().getconfig().get('required_metadata', [])
for key, comment, dtype, allowed in reqmd:
if key in columns and allowed:
selectOptions[key] = allowed
return render_template("datatable.html",
columns=columns,
data=[],
selectOptions=selectOptions)
return app
(index, item): index - item):
group = map(itemgetter(1), group)
if len(group) > 1:
ranges.append('{0}-{1}'.format(group[0], group[-1]))
else:
ranges.append('{}'.format(group[0]))
return ','.join(ranges)
##### CONSTRUCT A BROWSER
from Browser import Browser
BROWSER = Browser()
##### CONSTRUCT EXECUTOR
from Executor import Executor
EXECUTOR = Executor()
##### LOGGING TOOLS
class LogFormatter(logging.Formatter):
"Custom formatter for logging INFO records in different format to warnings and errors"
def format(self, record):
#compute s according to record.levelno
#for example, by setting self._fmt
#according to the levelno, then calling
#the superclass to do the actual formatting
if record.levelno == logging.INFO:
self._fmt = "%(levelname)s\t%(message)s"
else:
self._fmt = "###\t%(levelname)s (%(name)s)\t%(message)s"
def __init__(self, schedule_parser, report_directory, schedule_name, report_sql_file, cs_id, tr_id, rq_param):
Executor.__init__(self, schedule_parser, report_directory, schedule_name, report_sql_file, cs_id, tr_id, rq_param)
self.AllProcess = []
self.should_stop = False
def __init__(self, schedule_parser, report_directory, schedule_name, report_sql_file, cs_id, tr_id, rq_param):
Executor.__init__(self, schedule_parser, report_directory, schedule_name, report_sql_file, cs_id, tr_id, rq_param)
from Executor import Executor
import sys
if __name__ == '__main__':
if len(sys.argv) < 2:
print 'Usage: {} <file> [func]'.format(sys.argv[0])
sys.exit()
file = sys.argv[1]
if len(sys.argv) == 2:
func = 'main'
else:
func = sys.argv[2]
ex = Executor(file, func)
ex.run()
def _init_executors(self, num):
self._logger.info("init {num} executors".format_map(vars()))
self._executors = [Executor.start() for _ in range(num)]
self._num_executors = len(self._executors)
PORT = 5005
TYPE_JSON = 1;
TYPE_MESSAGE = 2;
cmd_q = Queue.Queue()
activeUser = {}
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.bind((HOST, PORT))
except socket.error as msg:
print 'Bind failed. Error Code : ' + str(msg[0]) + ' Message ' + msg[1]
sys.exit()
executor = Executor(cmd_q)
executor.start()
s.listen(10)
print 'Socket now listening'
#Function for handling connections. This will be used to create threads
def clientthread(conn, queue):
#Sending message to connected client
conn.send('CONN_ACC') #send only takes string
start_new_thread(clientInThread, (conn, queue))
def clientInThread(conn, in_q):
out_q = Queue.Queue()
start_new_thread(clientOutThread, (conn, out_q))