def run(host, port, config_filename):
"""
Runs the server.
@param host The host for the server
@param port The port for the server
"""
import config
config.load_config_file(config_filename)
global triangulation, detectionserver, fingerprint
import localization
import detectionserver
import fingerprint
import db
import detectionserver
import pageserver
db.init()
config.app.run(host=host, port=int(port), debug=True)
def run(host, port, config_filename):
"""
Runs the server.
@param host The host for the server
@param port The port for the server
"""
import config
config.load_config_file(config_filename)
global triangulation, detectionserver, fingerprint
import localization
import detectionserver
import fingerprint
import db
import detectionserver
import pageserver
db.init()
config.app.run(host=host, port=int(port), debug=True)
def init_preferences_widgets(self):
'''初始化首选项对话框'''
self.file_chooser_dialog = gtk.FileChooserDialog("选择一个文件", None,
gtk.FILE_CHOOSER_ACTION_OPEN,
(gtk.STOCK_OPEN, gtk.RESPONSE_OK,
gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL))
self.folder_chooser_dialog = gtk.FileChooserDialog("选择一个文件夹", None,
gtk.FILE_CHOOSER_ACTION_SELECT_FOLDER,
(gtk.STOCK_OPEN, gtk.RESPONSE_OK,
gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL))
load_config_file()
self.update_player_widgets()
self.update_preferences_widgets()
def run(host, port, config_file):
import config
config.load_config_file(config_file)
global triangulation, detectionserver, fingerprint
import localize
import remserver
import fingerprint
import db
import detectionserver
import pageserver
db.init()
config.app.run(host=host, port=int(port), debug=True)
def check_data(obsid, beg=None, dur=None, base_dir=None):
if base_dir is None:
comp_config = config.load_config_file()
base_dir = comp_config['base_data_dir']
comb_dir = "{0}{1}/combined".format(base_dir, obsid)
if type(beg) is not int:
beg = int(beg)
if type(dur) is not int:
dur = int(dur)
#Check to see if the files are combined properly
if beg is not None and dur is not None:
logger.info("Checking recombined files beginning at {0} and ending at {1}. Duration: {2} seconds"\
.format(beg, (beg+dur), dur))
error = checks.check_recombine(obsid,
startsec=beg,
n_secs=dur,
directory=comb_dir)
else:
logger.warn(
"No start time information supplied. Comparing files with full obs"
)
error = checks.check_recombine(obsid, directory=comb_dir)
if error == True:
logger.error("Recombined files check has failed. Cannot continue")
sys.exit(1)
else:
logger.info("Recombined check passed")
return
def KeyPressEvent(self):
cmd = Command.lookup(self._event_data['keycode'], self._event_data['modifiers'])
if not cmd:
return
x = cmd.get_global_command()
if x == 'quit':
for tiler in state.iter_tilers():
tiler.cmd_untile()
self._stop = True
elif x == 'debug':
state.print_hierarchy(*state.get_active_wsid_and_mid())
elif x == 'refresh_workarea':
state.update_property('_NET_WORKAREA')
elif x == 'reload_configuration':
config.load_config_file()
state.update_NET_DESKTOP_GEOMETRY(True)
state.apply_config()
else:
Tile.dispatch(state.get_active_monitor(), cmd)
def lambda_handler(event, context):
dic = config.load_config_file(config_bucket, config_key_name)
rss_config = RSSConfig.of(dic['functions']['collect_rss_entries'])
global logger
logger = log.get_logger(dic['globals']['log_level'])
new_entry = 0
for rss_config_item in rss_config.get_items():
try:
res = feedparser.parse(rss_config_item.get_url())
new_entry += handle_entries(res.entries, rss_config_item, topic,
rss_config.get_format())
except Exception as e:
logger.error(e)
continue
return {'new_entry': new_entry}
def main():
config.check_config_file()
app_config = config.load_config_file()
pushbullet_api_key = app_config.get('api_key')
regex_to_test = getattr(args, 'regex', app_config.get('regex'))
if regex_to_test:
regex_to_test = re.compile(regex_to_test)
site_url = getattr(args, 'url', app_config.get('site_to_parse'))
if getattr(args, 'mode', False):
match_if_present = False if getattr(args, 'mode', None) == 'NOT_FOUND' else True
else:
match_if_present = False if app_config.get('match_mode') == 'NOT_FOUND' else True
if not pushbullet_api_key or not regex_to_test or not site_url:
print "Some config values or arguments not given. Exiting..."
exit(-1)
while True:
print "\n", datetime.now()
response = get_response_from_site(site_url)
site_content = BeautifulSoup(response.text)
page_title = site_content.find('title').text
matching_content = site_content.find_all(text=regex_to_test)
matching_content += site_content.find_all('a', href=regex_to_test)
if match_if_present:
if matching_content:
print REGEX_FOUND_MESSAGE
send_notification(pushbullet_api_key, page_title, site_url)
break
print REGEX_NOT_FOUND_MESSAGE
else:
if not matching_content:
print REGEX_NOT_FOUND_MESSAGE
send_notification(pushbullet_api_key, page_title, site_url)
break
print REGEX_FOUND_MESSAGE
sleep(10)
return
def lambda_handler(event, context):
dic = config.load_config_file(config_bucket, config_key_name)
website_config = WebsiteConfig.of(
dic['functions']['detect_website_changes'])
global logger
logger = log.get_logger(dic['globals']['log_level'])
changed = 0
error = 0
for item in website_config.get_items():
try:
website_changed = check_if_website_updated(
item, website_config.get_format())
if website_changed:
changed += 1
except:
print(traceback.format_exc())
error += 1
return {'changed': changed, 'error': error}
def creat_model(self, config_file="ner.config"):
from config import load_config_file
task_dict, hyper_parameters_dict, extra_parameters_dict, = load_config_file(
configFile=config_file)
u""" 读取预处理的word2id文件（实际上是每个字分配一个id) """
word2id = read_dictionary(
os.path.join('embeddings', extra_parameters_dict["word2id"]))
## paths setting
""" 处理对模型结果等文件的保存名字及路径, 以及logger的保存位置 """
ckpt_file = tf.train.latest_checkpoint(self.model_path)
self.model = ZHNer(hyper_parameters_dict,
tag2label,
word2id,
config=config)
self.saver = tf.train.import_meta_graph(ckpt_file + ".meta")
self.sess = tf.Session(config=config)
self.saver.restore(self.sess, ckpt_file)
print("--------- INFO:checkpoint还原模型完成！,checkpoint = {}".format(
self.model_path))
def submit_dspsr(run_params):
#run dspsr
launch_line = "stokes_fold.py -m f -d {0} -p {1} -b {2} -s {3} -L {4} --mwa_search {5}\
--vcs_tools {6}"\
.format(run_params.pointing_dir, run_params.pulsar, run_params.nbins,\
run_params.subint, run_params.loglvl, run_params.mwa_search,\
run_params.vcs_tools)
if run_params.stop == True:
launch_line += " -S"
commands = []
commands.append("psrcat -e {0} > {1}/{0}.eph".format(
run_params.pulsar, run_params.pointing_dir))
commands.append("echo 'Running DSPSR folding...\n'")
commands.append("dspsr -cont -U 4000 -A -L {0} -E {3}/{1}.eph -K -b {2} -O {3}/{1}_subint_{0} {3}/*.fits"\
.format(run_params.subint, run_params.pulsar, run_params.nbins, run_params.pointing_dir))
commands.append("echo 'Attempting to find rotation measure.\nOutputting result to {0}/{1}_rmfit.txt\n'"\
.format(run_params.pointing_dir, run_params.pulsar))
commands.append("rmfit {0}/{1}_subint_{2}.ar -t > {0}/{1}_rmfit.txt"\
.format(run_params.pointing_dir, run_params.pulsar, run_params.subint))
#rerun the script
commands.append(launch_line)
name = "dspsr_RM_{0}_{1}".format(run_params.pulsar, run_params.obsid)
comp_config = config.load_config_file()
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "08:00:00"},\
module_list=["mwa_search/{0}".format(run_params.mwa_search),\
"dspsr/master", "psrchive/master"],\
submit=True, vcstools_version=run_params.vcs_tools)
logger.info("Job submitted for dspsr using\n\
pointing directory: {0}\n\
pulsar: {1}"\
.format(run_params.pointing_dir, run_params.pulsar))
def submit_RM_correct(run_params):
#correct for RM and submit plot
launch_line = "stokes_fold.py -m p -d {0} -p {1} -b {2} -s {3} -L {4} --mwa_search {5}\
--vcs_tools {6}"\
.format(run_params.pointing_dir, run_params.pulsar, run_params.nbins,\
run_params.subint, run_params.loglvl, run_params.mwa_search,\
run_params.vcs_tools)#, run_params.stop)
if run_params.stop == True:
launch_line += " -S"
commands = []
#correct for RM
commands.append("echo 'Correcting for input rotation measure\n'")
commands.append("pam -e ar2 -R {0} {1}/{2}_subint_{3}.ar"\
.format(run_params.RM, run_params.pointing_dir, run_params.pulsar, run_params.subint))
#Turn the archive into a readable ascii file
commands.append("echo 'Wiritng result to text file\n'")
commands.append("pdv -FTt {0}/{1}_subint_{2}.ar2 > {0}/{1}_archive.txt".\
format(run_params.pointing_dir, run_params.pulsar, run_params.subint))
#launch plotting
commands.append(launch_line)
name = "RMcor_plt_{0}_{1}".format(run_params.pulsar, run_params.obsid)
comp_config = config.load_config_file()
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "02:00:00"},\
module_list=["mwa_search/{0}".format(run_params.mwa_search),
"psrchive/master"],\
submit=True, vcstools_version=run_params.vcs_tools)
################################################## # ____ ________ ________ # # / __ \___ ___ ___ / ___/ _ \/ __/ __/ # # / /_/ / _ \/ -_) _ \/ (_ / ___/\ \_\ \ # # \____/ .__/\__/_//_/\___/_/ /___/___/ # # /_/ by NotSoOld, 2017 (c) # # # # route|process|gather stats # # # # OpenGPSS Interpreter.py - starts all action! # # # ################################################## from modules import interpreter, errors import os import config config.load_config_file() interpreter.print_logo() print 'name of file with system to simulate:' f = raw_input() filepath = os.path.dirname(os.path.abspath(__file__))+'/'+f+'.ogps' if not os.path.isfile(filepath): errors.print_error(1, '', [filepath]) interpreter.start_interpreter(filepath)
def submit_prepfold(run_params, nbins=100, finish=False):
if nbins is not int:
nbins = int(float(nbins))
comp_config = config.load_config_file()
#Check to see if there is a 100 bin fold already
bin_lim = bin_sampling_limit(run_params.pulsar)
if len(glob.glob("*_100_bins**{0}*bestprof".format(
run_params.pulsar))) > 0 and bin_lim > 100 and nbins is not 100:
#add a prepfold command for 100 bins
logger.info("Folding on 100 bins for pointing {0}".format(
run_params.pointing))
commands = []
commands = add_to_prepfold(commands,
run_params.pointing_dir,
run_params.pulsar,
run_params.obsid,
nbins=100)
name = "binfinder_prepfold_only_{0}_100".format(run_params.pulsar)
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "2:00:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
'presto/no-python'],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
logger.info("Prepfold job successfully submitted: {0}".format(name))
launch_line = "binfinder.py -d {0} -t {1} -O {2} -o {3} -L {4} --prevbins {5} --vcs_tools {6}\
--mwa_search {7} -p {8}"\
.format(run_params.pointing_dir, run_params.threshold, run_params.cal_id,\
run_params.obsid, run_params.loglvl, nbins, run_params.vcs_tools,\
run_params.mwa_search, run_params.pulsar)
if run_params.stop == True:
launch_line += " -S"
logger.info("Submitting job for {0} bins".format(nbins))
#create slurm job:
commands = []
if firstrun == True:
commands = add_to_prepfold(commands,
run_params.pointing_dir,
run_params.pulsar,
run_params.obsid,
nbins=100)
commands = add_prepfold_to_commands(commands,
run_params.pointing_dir,
run_params.pulsar,
run_params.obsid,
nbins=nbins)
if finish == False:
#Rerun this script
commands.append(
'echo "Running script again. Passing prevbins = {0}"'.format(
nbins))
launch_line += " -m f"
else:
#Run again only once and without prepfold
commands.append(
'echo "Running script again without folding. Passing prevbins = {0}"'
.format(nbins))
launch_line += " -m e"
commands.append(launch_line)
comp_config = config.load_config_file()
name = "binfinder_{0}_{1}".format(run_params.pulsar, nbins)
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "2:00:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
'presto/no-python'],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
logger.info("Job successfully submitted: {0}".format(name))
def submit_multifold(run_params, nbins=100):
job_ids = []
comp_config = config.load_config_file()
#Check beam coverage for the pulsar
start, end = pulsar_beam_coverage(run_params.obsid, run_params.pulsar)
logger.info(
"start and end of pulsar beam coverage for on-disk files:{0}, {1}".
format(start, end))
if start >= 1. or end < 0.:
logger.error(
"pulsar is not in beam for any of the on-disk files. Ending...")
sys.exit(1)
for i, pointing in enumerate(run_params.pointing_dir):
logger.info("submitting pointing:{0}".format(pointing))
#os.chdir(pointing)
#create slurm job:
commands = []
commands = add_prepfold_to_commands(commands, pointing, run_params.pulsar, run_params.obsid,\
start=start, end=end, nbins=nbins)
name = "multifold_binfind_{0}_{1}".format(run_params.pulsar, i)
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
myid = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "2:00:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
'presto/no-python'],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
job_ids.append(myid)
#Now submit the check script
if run_params.stop == True:
stop = "-S"
else:
stop = ""
p = ""
for pointing in run_params.pointing_dir:
p += " " + pointing
commands = []
commands.append("binfinder.py -m b -d {0} -O {1} -p {2} -o {3} -L {4} {5} --vcs_tools {6}\
--mwa_search {7} --force_initial -p {8}"\
.format(p, run_params.cal_id, run_params.pulsar, run_params.obsid, run_params.loglvl,\
stop, run_params.vcs_tools, run_params.mwa_search, run_params.pulsar))
name = "best_pointing_{0}".format(run_params.pulsar)
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
myid = submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "00:30:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search),\
"presto/no-python"],\
submit=True, depend=job_ids, depend_type="afterany",\
vcstools_version="master")
def find_fwhm_and_plot(obsid, pointing):
pointing_list = []
sn = []
comp_config = config.load_config_file()
for d in glob.glob("{0}/{1}/pointings/*".format(comp_config['base_product_dir'],
obsid)):
bestprof_file = glob.glob("{0}/{1}*_PSR_2330-2005.pfd.bestprof".format(d, obsid))
if len(bestprof_file) == 1:
with open(bestprof_file[0]) as bestfile:
for i in bestfile.readlines():
if i.startswith("# Prob(Noise)"):
pointing_list.append(d.split("/")[-1])
sn.append(float(i.split("(")[-1][1:-7]))
#find max for a FWHM test
#max_index = sn.index(max(sn))
ra_hex = pointing.split("_")[0]
dec_hex = pointing.split("_")[1]
print(ra_hex, dec_hex)
coord = SkyCoord(ra_hex,dec_hex,unit=(u.hourangle,u.deg))
dec_centre = coord.dec.degree
ra_centre = coord.ra.degree
ras = []; decs = []
ra_line = []; ra_sn_line = []
dec_line = []; dec_sn_line = []
print(sn)
for i in range(len(sn)):
rah, dech = pointing_list[i].split("_")
coord = SkyCoord(rah,dech,unit=(u.hourangle,u.deg))
ras.append(coord.ra.degree)
decs.append(coord.dec.degree)
if decs[i] == dec_centre:
ra_line.append(ras[i])
ra_sn_line.append(sn[i])
if ras[i] == ra_centre:
dec_line.append(decs[i])
dec_sn_line.append(sn[i])
max_ra_i = np.argmax(ra_sn_line)
max_dec_i = np.argmax(dec_sn_line)
max_coord = SkyCoord(ra_line[max_ra_i], dec_line[max_dec_i],unit=(u.deg,u.deg))
ra_max_hex = max_coord.ra.to_string(unit=u.hour, sep=':')
dec_max_hex = max_coord.dec.to_string(unit=u.degree, sep=':')
print("sn max coord: {0}_{1}".format(ra_max_hex, dec_max_hex))
#sort and calc FWHM
ra_sn_line = [x for _,x in sorted(zip(ra_line,ra_sn_line))]
ra_line = sorted(ra_line)
print(ra_sn_line,ra_line)
dec_sn_line = [x for _,x in sorted(zip(dec_line,dec_sn_line))]
dec_line = sorted(dec_line)
print(dec_sn_line,dec_line )
ra_sn_line = np.array(ra_sn_line) ; dec_sn_line = np.array(dec_sn_line)
spline = UnivariateSpline(ra_line, ra_sn_line-np.max(ra_sn_line)/2., s=0)
print(spline.roots())
if len(spline.roots()) == 2:
r1, r2 = spline.roots()
ra_FWHM = abs(r1-r2)
print("raw ra FHWM: " + str(ra_FWHM))
cor_ra_FWHM = float(ra_FWHM)*math.cos(np.radians(dec_centre))
print("corrected ra FWHM: {0}".format(cor_ra_FWHM))
else:
print("No detectable ra FWHM (too many roots)")
spline = UnivariateSpline(dec_line, dec_sn_line-np.max(dec_sn_line)/2., s=0)
if len(spline.roots()) == 2:
r1, r2 = spline.roots()
dec_FWHM = abs(r1-r2)
print("raw dec FHWM: " + str(dec_FWHM))
cor_dec_FWHM = float(dec_FWHM)*math.cos(np.radians(dec_centre) + np.radians(26.7))**2
print("corrected dec FWHM: {0}".format(cor_dec_FWHM))
else:
print("No detectable dec FWHM (too many roots)")
diff = 10**20
# Find the min diff by comparing difference
# of all possible pairs in given array
n = len(dec_line)
for i in range(n-1):
for j in range(i+1,n):
if abs(dec_line[i]-dec_line[j]) < diff:
diff = abs(dec_line[i] - dec_line[j])
n = len(ra_line)
for i in range(n-1):
for j in range(i+1,n):
if abs(ra_line[i]-ra_line[j]) < diff:
diff = abs(ra_line[i] - ra_line[j])
diff = 0.01
print("Diff: {}".format(diff))
ras = np.array(ras); decs = np.array(decs)
fig = plt.figure(figsize=(7, 7))
ax = fig.add_subplot(111)
plt.grid(True)
#sort by sn
ras = [x for _,x in sorted(zip(sn,ras))]
decs = [x for _,x in sorted(zip(sn,decs))]
sn = sorted(sn)
cm = plt.cm.get_cmap('plasma',20)
ax.grid(color='r', linestyle='-', linewidth=2)
sp = plt.scatter(ras, decs, c=sn, s=(1000*diff)**2, cmap = cm)
#plt.gray()
plt.colorbar(sp)
plt.savefig("{0}_position_heatmap.png".format(obsid))
plt.show()
(epoch, num_epochs, epoch_train_loss, epoch_train_accuracy,
epoch_end - epoch_start))
# outer for loop ends
return
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('exp_name', nargs='?', default='PeterUnetInception2')
args = parser.parse_args()
exp_name = args.exp_name
cfg = config.load_config_file(exp_name)
# train_data_loader, train_tile_borders = get_small_loader(
train_data_loader, train_tile_borders = get_train_loader(
cfg['train']['batch_size'], cfg['train']['paddings'],
cfg['train']['tile_size'], cfg['train']['hflip'],
cfg['train']['shift'], cfg['train']['color'], cfg['train']['rotate'],
cfg['train']['scale'], cfg['train']['fancy_pca'],
cfg['train']['edge_enh'])
# val_data_loader, val_tile_borders = get_small_loader(
val_data_loader, val_tile_borders = get_val_loader(
cfg['test']['batch_size'], cfg['test']['paddings'],
cfg['test']['tile_size'], False, False, False, False, False, False,
False)
trainer(exp_name,
import numpy
from transformation import *
from geometry import *
from timing import Timing
import config
from context import Context
from drawingutils import *
parser = argparse.ArgumentParser()
parser.add_argument("configuration_filepath",
metavar="CONFIG_FILE",
nargs='?',
help="JSON configuration file to load")
options = parser.parse_args()
config = config.load_config_file(options.configuration_filepath)
compor_cena = None
processar_teclado = None
try:
student_module = importlib.import_module(config.module_name)
compor_cena = getattr(student_module, config.callback_name)
processar_teclado = getattr(student_module, "processar_teclado", None)
except ImportError:
print("*** Atencao: Arquivo %s.py nao foi encontrado." %
config.module_name,
file=sys.stderr)
except AttributeError:
print("*** Atencao: Arquivo %s.py nao possui funcao '%s'." %
(config.module_name, config.callback_name),
def api_get_disk_info(disk='/'):
if not disk.startswith('/'):
disk = '/' + disk
ctx = api.get_disk_info(disk=disk)
return Response(response=json_dumps(ctx), mimetype="application/json")
@app.route('/api/process/', defaults={'process_filter': PROCESSES})
@app.route('/api/process/<process_filter>')
def api_get_filtered_processes(process_filter=None):
if not process_filter:
process_filter = PROCESSES if PROCESSES else ''
process_filter = process_filter.split(',')
ctx = api.get_filtered_processes(process_filter)
return Response(response=json_dumps(ctx), mimetype="application/json")
if __name__ == '__main__':
config.check_config_file()
app_config = config.load_config_file()
PROCESSES = app_config.get('processes')
DISKS = app_config.get('disks', '').split(',')
app.run(threaded=True, host='0.0.0.0', debug=True)
'--num_threads',
type=int,
help=
'number of parallel threads running in PPO (0=automatic number of cpus)',
default=0)
parser.add_argument(
'-t',
'--thread',
action="store_true",
help='do not use: technical parameter for parallel run')
args = parser.parse_args()
if args.gpus:
args.gpus = [int(s) for s in args.gpus.split(',')]
torch.cuda.set_device(args.gpus[0])
config = load_config_file(args.algorithm)
experiment = Config(config[args.env][str(args.config)],
"{0}_{1}".format(args.env, str(args.config)))
update_config(args, experiment)
if args.load != '':
env_class = set_env_class(args.algorithm, args.env, experiment)
env_class.test(experiment, args.load)
else:
if args.thread:
experiment.trials = 1
if args.parallel:
if args.num_processes == 0:
num_cpus = psutil.cpu_count(logical=True)
from rev import CANSparkMax, MotorType
from ctre import WPI_TalonSRX
import navx
from components.limelight import Limelight
from components.navx import NavX
from components.neo import Neo
from components.shooter import Shooter
from config import load_config_file
from dash import put_tuple
from oi import DriverController, SysopController
config_data = load_config_file()
subsystems_enabled = {
"limelight": "limelight" in config_data['enabled_devices'],
"navx": "navx" in config_data['enabled_devices'],
"shooter": "shooter" in config_data['enabled_devices'],
"neo": "neo" in config_data['enabled_devices'],
"camera": "camera" in config_data['enabled_devices']
}
class Robot(magicbot.MagicRobot):
if subsystems_enabled['limelight']:
limelight_component: Limelight
if subsystems_enabled['navx']:
navx_component: NavX
parser.add_argument("--consume", dest='consume_videos', help="override the ini conf to consume videos", action="store_true")
parser.add_argument("--no-consume", dest='consume_videos', help="override the ini conf to consume videos", action="store_false")
parser.set_defaults(consume_videos=None)
parser.add_argument("--speed", type=int, metavar='6', help="override the ini conf for the speed of the simulation")
parser.add_argument("--out", metavar='/path/to/stats', help="override the ini conf for the output folder")
parser.add_argument("--proxy", metavar='FIFOProxy', help="override the ini conf for the proxy to use")
parser.add_argument("--parallel", help="use true parallelism when comparing", action="store_true")
parser.set_defaults(parallel=False)
parser.add_argument("--compare-to", dest='proxy2', metavar='LRUProxy', help="compare the first proxy to this one")
args = parser.parse_args()
if args.verbosity:
print("verbosity turned on")
if args.config:
if not config.load_config_file(args.config):
sys.exit()
else:
if not config.load_config_file():
sys.exit()
if args.skip_inactivity != None:
config.set_skip_activity(args.skip_inactivity)
if args.consume_videos != None:
config.set_consume_videos(args.consume_videos)
if args.trace:
config.set_trace_file(args.trace)
if args.db:
config.set_db_file(args.db)
def submit_slurm(name,
commands,
tmpl=SLURM_TMPL,
slurm_kwargs=None,
module_list=[],
vcstools_version="master",
batch_dir="batch/",
depend=None,
depend_type='afterok',
submit=True,
outfile=None,
queue="cpuq",
export="NONE",
gpu_res=None,
mem=1024,
cpu_threads=1,
temp_mem=None,
nice=0,
shebag='#!/bin/bash -l'):
"""
Making this function to cleanly submit SLURM jobs using a simple template.
Parameters
----------
name : str
The base name that is used to create the "`name`.batch" and "`name`.out" files.
commands : list of strs
The actual bash script commands you wnat to run.
Expects a list where each element is a single line of the bash script.
tmpl : str
A template header string with format place holders: export, outfile,
cluster, header and script.
This is used to create the final string to be written to the job script.
For this function, it is required to be SLURM compliant.
Default: `SLURM_TMPL`
slurm_kwargs : dict [optional]
A dictionary of SLURM keyword, value pairs to fill in whatever is not
in the template supplied to `tmpl`.
Default: `{}` (empty dictionary, i.e. no additional header parameters)
module_list : list of str [optional]
A list of module names (including versions if applicable) that will
be included in the header for the batch
scripts. e.g. ["vcstools/master", "mwa-voltage/master", "presto/master"] would append
module load vcstools/master
module load mwa-voltage/master
module load presto/master
to the header of the batch script. This can also invoke "module use ..." commands.
NOTE: /group/mwa/software/modulefiles is used and vcstools/master is loaded by default.
vcstools_version : str
The version of vcstools to load. Default: master.
batch_dir : str [optional]
The LOCAL directory where you want to write the batch scripts
(i.e. it will write to `$PWD/batch_dir`).
Default: "batch/"
depend : list or None [optional]
A list of the SLURM job IDs that your would like this job to depend on.
If `None` then it is assumed there is no dependency on any other job.
Default: `None`
depend_type : str [optional]
The type of slurm dependancy required. For example if you wanted the
job to run after the jobs have been terminated use 'afterany'.
Default: "afterok"
submit : boolean [optional]
Whether to write and submit the job scripts (`True`) or only write the scripts (`False`).
Default: `True`
outfile : str [optional]
The output file name if "`name`.out" is not desirable.
Default: `None` (i.e. "`batch_dir`/`name`.out")
queue : str [optional]
The type of queue you require (cpuq, gpuq or copyq) then the script will
choose the correct partitions and clusters for the job to run on
Default: "cpuq"
export : str [optional]
Switch that lets SLURM use your login environment on the compute
nodes ("ALL") or not ("NONE").
Default: "None"
gpu_res : int [optional]
Number of GPUs that the SLURM job will reserve.
Default: "None"
mem : int [optional]
The MB of ram required for your slurm job.
Default: 8192
cpu_threads : int [optional]
The number of cpu threads required for your slurm job.
Default: 1
Returns
-------
jobid : int
The unique SLURM job ID associated with the submitted job.
"""
if slurm_kwargs is None:
slurm_kwargs = {}
#Work out which partition and cluster to use based on the supercomputer
#(in config file) and queue required
comp_config = config.load_config_file()
if queue == 'cpuq':
cluster = comp_config['cpuq_cluster']
partition = comp_config['cpuq_partition']
elif queue == 'gpuq':
cluster = comp_config['gpuq_cluster']
partition = comp_config['gpuq_partition']
if gpu_res is None:
# No gpus reserved so change it to a default of 1
gpu_res = 1
elif queue == 'copyq':
cluster = comp_config['copyq_cluster']
partition = comp_config['copyq_partition']
elif queue == 'zcpuq':
# Download and checks should be done on Zeus's cpuq. This will only work
# on Galaxy as the Ozstar workflow is different
cluster = comp_config['zcpuq_cluster']
partition = comp_config['zcpuq_partition']
else:
logger.error("No queue found, please use cpuq, gpuq or copyq")
header = []
if batch_dir.endswith("/") is False:
batch_dir += "/"
# define file names (both the batch job file and the output file)
jobfile = batch_dir + name + ".batch"
if not outfile:
outfile = batch_dir + name + ".out"
# create the header from supplied arguments
for k, v in slurm_kwargs.items():
if len(k) > 1:
k = "--" + k + "="
else:
k = "-" + k + " "
header.append("#SBATCH {0}{1}".format(k, v))
# check if there are dependencies, and if so include that in the header
if depend is not None:
#assumes append is a list but if not will make an educated guess of how to reformat it
if isinstance(depend, int):
#assume it's ben given a single job id
header.append("#SBATCH --dependency={0}:{1}".format(
depend_type, depend))
if isinstance(depend, str):
if ":" in depend:
#assume it has been given an already formated string
if depend.startswith(":"):
depend = depend[1:]
#or a single jobid
header.append("#SBATCH --dependency={0}:{1}".format(
depend_type, depend))
if isinstance(depend, list):
depend_str = ""
for job_id in depend:
depend_str += ":" + str(job_id)
header.append("#SBATCH --dependency={0}{1}".format(
depend_type, depend_str))
# add a gpu res to header
if gpu_res is not None:
header.append('#SBATCH --gres=gpu:{0}'.format(gpu_res))
# add temp SSD memory to combat I/O issues. Only availble on Ozstar
hostname = socket.gethostname()
if temp_mem is not None and \
(hostname.startswith('john') or hostname.startswith('farnarkle')):
header.append("#SBATCH --tmp={0}GB".format(temp_mem))
# now join the header into one string
header = "\n".join(header)
# construct the module loads
modules = []
switches = []
for m in module_list:
if m == "vcstools":
# don't do anything as vcstools is loaded automatically
continue
if "module switch" in m:
# if a module switch command is included rather than just a module name, then add it to a separate list
switches.append(m)
elif "module" in m:
modules.append("{0}\n".format(m))
else:
modules.append("module load {0}\n".format(m))
# join the module loads and switches into a single string
switches = "\n".join(switches)
modules = "\n".join(modules)
# join the commands into a single string
commands = "\n".join(commands)
# load computer dependant config file
comp_config = config.load_config_file()
# some little hacks to make jobs work on the shanghai server
if hostname.startswith('x86') or hostname.startswith('arm'):
if vcstools_version == 'master':
vcstools_version = 'cpu-master'
if export == "NONE":
export = "ALL"
if shebag == "#!/bin/bash -l":
shebag = "#!/bin/bash"
# format the template script
tmpl = tmpl.format(shebag=shebag,
script=commands,
outfile=outfile,
header=header,
switches=switches,
modules=modules,
version=vcstools_version,
cluster=cluster,
partition=partition,
export=export,
account=comp_config['group_account'],
module_dir=comp_config['module_dir'],
threads=cpu_threads,
mem=mem,
nice=nice)
# write the formatted template to the job file for submission
with open(jobfile, "w") as fh:
fh.write(tmpl)
# submit the jobs
batch_submit_line = "sbatch {0}".format(jobfile)
jobid = None
if submit:
submit_cmd = subprocess.Popen(batch_submit_line,
shell=True,
stdout=subprocess.PIPE)
for line in submit_cmd.stdout:
if b"Submitted" in line:
jobid = str(line.split(b" ")[3].decode())
if jobid is None:
logger.debug(batch_submit_line)
logger.debug(submit_cmd.stdout)
return
else:
return jobid
else:
return
def add_prepfold_to_commands(commands,
pointing,
pulsar,
obsid,
use_mask=True,
start=None,
end=None,
nbins=100,
ntimechunk=120,
dmstep=1,
period_search_n=1):
#find the beginning and end of the pulsar's beam coverage for this obs
if start == None or end == None:
start, end = pulsar_beam_coverage(obsid, pulsar)
logger.info(
"start and end of pulsar beam coverage for on-disk files:{0}, {1}".
format(start, end))
if start >= 1. or end < 0.:
logger.error(
"pulsar is not in beam for any of the on-disk files. Ending..."
)
sys.exit(1)
comp_config = config.load_config_file()
#Figure out whether or not to input a mask
if use_mask == True:
check_mask = glob.glob("{0}{1}/incoh/*.mask".format(
comp_config['base_product_dir'], obsid))
if check_mask:
mask = "-mask " + check_mask[0]
else:
mask = ""
else:
mask = ""
#make the prepfold command
constants = "-pstep 1 -pdstep 2 -ndmfact 1 -noxwin -nosearch -runavg -noclip -nsub 256 1*fits "
variables = "-o {0}_{1}_bins ".format(obsid, nbins)
variables += mask
variables += "-n {0} ".format(nbins)
variables += "-start {0} -end {1} ".format(start, end)
variables += "-dmstep {0} ".format(dmstep)
variables += "-npart {0} ".format(ntimechunk)
variables += "-npfact {0} ".format(period_search_n)
#load presto module here because it uses python 2
commands.append('cd {0}'.format(pointing))
commands.append('echo "Folding on known pulsar {0}"'.format(pulsar))
commands.append('psrcat -e {0} > {0}.eph'.format(pulsar))
commands.append("sed -i '/UNITS TCB/d' {0}.eph".format(pulsar))
commands.append("prepfold -timing {0}.eph {1} {2}"\
.format(pulsar, variables, constants))
commands.append('errorcode=$?')
commands.append('pulsar={}'.format(pulsar[1:]))
pulsar_bash_string = '${pulsar}'
#Some old ephems don't have the correct ra and dec formating and
#causes an error with -timing but not -psr
commands.append('if [ "$errorcode" != "0" ]; then')
commands.append(' echo "Folding using the -psr option"')
commands.append(' prepfold -psr {0} {1} {2}'\
.format(pulsar, variables, constants))
commands.append(' pulsar={}'.format(pulsar))
commands.append('fi')
commands.append('rm {0}.eph'.format(pulsar))
return commands
def create_app(config_name):
global api, jwt
app = Flask(__name__)
app.config.from_object(Config())
load_config_file(app.config, config_name)
with app.app_context():
db.init_app(app)
db.app = app
jwt.init_app(app)
api = Api(app)
current_app = app
logging.basicConfig()
handler = RotatingFileHandler('idlak-server.log',
maxBytes=100000,
backupCount=1)
handler.setLevel(app.config['LOGGING'])
app.logger.addHandler(handler)
app.logger.setLevel(app.config['LOGGING'])
if not app.config['AUTHORIZATION']:
app.logger.warning('AUTHORIZATION is turned off, this should only ' +
'be used during debugging!!! Turn it back on in ' +
'the config file!')
with app.app_context():
from app import models, endpoints, reqlogging # noqa
from app.models.user import User # noqa
# if database is not created
if not os.path.isfile(app.config['DATABASE_NAME'] + '.db'):
db.create_all()
# check if there are any users, if there are none, create an admin
if len(User.query.all()) == 0:
admin_user = User.new_user_full('admin', 'admin', True)
app.logger.info(
"An initial admin user has been created: {}".format(
admin_user))
# url endpoints
from app.endpoints.auth import Auth, Auth_Expire
api.add_resource(Auth, '/auth')
api.add_resource(Auth_Expire, '/auth/expire')
from app.endpoints.language import Languages, Accents
api.add_resource(Languages, '/languages')
api.add_resource(Accents, '/languages/<lang_iso>/accents')
from app.endpoints.speech import Speech
api.add_resource(Speech, '/speech')
from app.endpoints.user import Users, Users_Password, Users_Delete, Toggle_Admin
api.add_resource(Users, '/users')
api.add_resource(Users_Password, '/users/<user_id>/password')
api.add_resource(Users_Delete, '/users/<user_id>')
api.add_resource(Toggle_Admin, '/users/<user_id>/admin')
from app.endpoints.voice import Voices, VoiceDetails
api.add_resource(Voices, '/voices')
api.add_resource(VoiceDetails, '/voices/<voice_id>')
return app
ch.setFormatter(formatter)
logger.addHandler(ch)
logger.propagate = False
#option parsing
if not args.obsid:
print("Please input observation id by setting -o or --obsid. Exiting")
quit()
if not args.cal_obs:
print(
"Please input calibration observation id by setting -O or --cal_obs. Exiting"
)
quit()
comp_config = config.load_config_file()
if not args.DI_dir:
args.DI_dir = "{0}/{1}/cal/{2}/rts/".format(
comp_config['base_product_dir'], args.obsid, args.cal_obs)
print("No DI_dir given so assuming {0} is the directory".format(
args.DI_dir))
if args.begin and args.end:
beg = args.begin
end = args.end
elif args.all:
beg, end = obs_max_min(args.obsid)
else:
find_beg_end(args.obsid, base_path=comp_config['base_product_dir'])
#Perform data checks
import logging
import os
import subprocess
from logging.handlers import RotatingFileHandler
from flask import Flask
from config import Config,load_config_file
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate, init as db_init, migrate as db_migrate, upgrade as db_upgrade, show
from flask_restful import Api, request
from flask_jwt_simple import JWTManager
from datetime import datetime
app = Flask(__name__)
app.config.from_object(Config())
load_config_file(app.config)
api = Api(app)
db = SQLAlchemy(app)
migrate = Migrate(app, db)
jwt = JWTManager(app)
logging.basicConfig()
handler = RotatingFileHandler('idlak-server.log', maxBytes=100000, backupCount=1)
handler.setLevel(logging.DEBUG)
app.logger.addHandler(handler)
app.logger.setLevel(logging.DEBUG)
from app import models, endpoints, reqlogging
from app.models.user import User
def submit_to_db(run_params, ideal_bins):
logger.info("submitting profile to database: {0}".format(
run_params.bestprof))
#Add path to filenames for submit script
cwd = os.getcwd()
ppps = cwd + "/" + glob.glob("*{0}_bins*{1}*.pfd.ps".format(
ideal_bins, run_params.pulsar[1:]))[0]
bestprof_name = cwd + "/" + glob.glob("*{0}_bins*{1}*.pfd.bestprof".format(
ideal_bins, run_params.pulsar[1:]))[0]
png_output = cwd + "/" + glob.glob("*{0}_bins*{1}*.png".format(
ideal_bins, run_params.pulsar[1:]))[0]
pfd = cwd + "/" + glob.glob("*{0}_bins*{1}*.pfd".format(
ideal_bins, run_params.pulsar[1:]))[0]
#do the same for 100 bin profiles
ppps_100 = cwd + "/" + glob.glob("*_100_bins*{0}*.pfd.ps".format(
run_params.pulsar[1:]))[0]
bestprof_name_100 = cwd + "/" + glob.glob(
"*_100_bins*{0}*.pfd.bestprof".format(run_params.pulsar[1:]))[0]
png_output_100 = cwd + "/" + glob.glob("*_100_bins*{0}*.png".format(
run_params.pulsar[1:]))[0]
pfd_100 = cwd + "/" + glob.glob("*_100_bins*{0}*.pfd".format(
run_params.pulsar[1:]))[0]
products = [ppps, bestprof_name, png_output, pfd,\
ppps_100, bestprof_name_100, png_output_100, pfd_100]
#move all of these data products to a suitable directory
data_dir = "/group/mwaops/vcs/{0}/data_products/{1}".format(
run_params.obsid, run_params.pulsar)
for product in products:
data_process_pipeline.copy_data(product, data_dir)
commands = []
commands.append('submit_to_database.py -o {0} --cal_id {1} -p {2} --bestprof {3} --ppps {4}'\
.format(run_params.obsid, run_params.cal_id, run_params.pulsar, bestprof_name, ppps))
commands.append(
'echo "submitted profile to database: {0}"'.format(bestprof_name))
if run_params.stop == False:
#Run stokes fold
commands.append("data_process_pipeline.py -d {0} -O {1} -p {2} -o {3} -b {4} -L {5}\
--mwa_search {6} --vcs_tools {7} -m s"\
.format(run_params.pointing_dir, run_params.cal_id, run_params.pulsar,\
run_params.obsid, run_params.best_bins, run_params.loglvl, run_params.mwa_search,\
run_params.vcs_tools))
#commands.append('echo "Searching for pulsar using the pipeline to test the pipelines effectivness"')
#commands.append('mwa_search_pipeline.py -o {0} -a --search --pulsar {1} -O {2}\
# --code_comment "Known pulsar auto test"'.format(run_params.obsid, run_params.pulsar,\
# run_params.cal_id))
name = "Submit_{0}_{1}".format(run_params.pulsar, run_params.obsid)
comp_config = config.load_config_file()
batch_dir = "{0}{1}/batch/".format(comp_config['base_product_dir'],
run_params.obsid)
submit_slurm(name, commands,\
batch_dir=batch_dir,\
slurm_kwargs={"time": "00:05:00"},\
module_list=['mwa_search/{0}'.format(run_params.mwa_search)],\
submit=True, vcstools_version="{0}".format(run_params.vcs_tools))
