def _write_rm_script(dupes_map):
"""
Format:
```
#! /bin/bash
# Keep: "/home/me/dupes/dir1/4.jpg"
rm "/home/me/dupes/dir2/4.jpg"
rm "/home/me/dupes/dir3/4.jpg"
# Keep: "/home/me/dupes/dir1/2.jpg"
rm "/home/me/dupes/dir2/2.jpg"
```
"""
utils.print_msg('\n> Writing rm script...')
if not dupes_map:
utils.print_msg('No dupes')
return
now = datetime.datetime.now()
path = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'rm_dupes_{}.sh'.format(now.strftime("%Y-%m-%d-%Hh%Mm")))
with open(path, 'w') as fout:
fout.write('#! /bin/bash\n')
fout.write('# >>>>> Extensions found in dupes: {}\n'.format(
' '.join(extensions)))
for check, paths in dupes_map.items():
to_keep = paths[0]
del paths[0]
fout.write('\n# Keep: "{}"\n'.format(os.path.join(root, to_keep)))
for to_remove in paths:
fout.write('rm "{}"\n'.format(os.path.join(root, to_remove)))
def _write_rm_script(self):
"""
Format:
```
#! /bin/bash
# Keep: "/home/me/dupes/dir1/4.jpg"
rm "/home/me/dupes/dir2/4.jpg"
rm "/home/me/dupes/dir3/4.jpg"
# Keep: "/home/me/dupes/dir1/2.jpg"
rm "/home/me/dupes/dir2/2.jpg"
```
"""
if not self.checksums_dupes_map:
return
now = datetime.datetime.now()
path = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'rm_dupes_{}.sh'.format(now.strftime("%Y-%m-%d-%Hh%Mm")))
utils.print_msg('> Writing rm script...')
with open(path, 'w') as fout:
fout.write('#! /bin/bash\n')
for check, paths in self.sizes_dupes_map.items():
to_keep = paths[0]
del paths[0]
fout.write('\n# Keep: "{}"\n'.format(
os.path.join(self.root, to_keep)))
for to_remove in paths:
fout.write('rm "{}"\n'.format(
os.path.join(self.root, to_remove)))
def wait_for_clients(self):
"""Wait for clients' request for connection and provide worker thread
serving the client.
"""
while True:
# Wait for client
client_conn, (client_ip, client_port) = self.socket.accept()
# The wait is done
with self._key_lock:
client_addr = client_ip + ":" + str(client_port)
if client_addr in self.client_conns:
self.client_conns[client_addr].close()
self.client_conns[client_addr] = client_conn
print_msg(client_addr + " connected")
print_msg("------------------------------------")
self.send_to_client(
{
'avg': self.model.fc.weight.data.clone(),
'seqnum': self.seqnum
}, client_conn, client_addr)
# Provide worker thread to serve client
worker_thread = threading.Thread(target=self.handle_request,
args=(client_conn, client_addr))
worker_thread.daemon = True
worker_thread.start()
def aggregate(self, ast_data):
self._seed_dict = {}
num_file = len(ast_data)
for i, v in enumerate(ast_data):
if v is None: continue
file_name, _frag_seq, _frag_info_seq, _node_types = v
frag_seq, frag_info_seq = [], []
msg = '[%d/%d] %s' % (i + 1, num_file, file_name)
print_msg(msg, 'INFO')
# Build node type list & dict
self.add_type(_node_types)
# Build fragment list, dict
for frag in _frag_seq:
frag_idx = self.add_frag(frag)
frag_seq += [frag_idx]
# Build dataset for additional features
for next_parent_idx, frag_type in _frag_info_seq:
next_parent_frag = frag_seq[next_parent_idx]
type_idx = self._type_dict[frag_type]
frag_info = (next_parent_frag, type_idx)
frag_info_seq += [frag_info]
self._seed_dict[file_name] = (frag_seq, frag_info_seq)
def set_sim_constants(boxsize_cMpc):
'''This method will set the values of relevant constants depending on the simulation
boxsize_cMpc is the box size in cMpc/h
It can be 37, 64, 114 or '''
global boxsize, LB, nbox_fine, M_box, M_grid, lscale, tscale, velconvert
#hf.print_msg('Setting constants for boxsize=%.3f cMpc' % boxsize_cMpc)
boxsize = boxsize_cMpc
LB = boxsize/const.h
#First argument is added by me
if utils.flt_comp(boxsize, 500.):
utils.print_msg('Setting conversion factors for 500/h Mpc box')
nbox_fine = 13824
elif utils.flt_comp(boxsize, 425.):
utils.print_msg('Setting conversion factors for 425/h Mpc box')
nbox_fine = 10976
elif utils.flt_comp(boxsize, 114.):
utils.print_msg('Setting conversion factors for 114/h Mpc box')
nbox_fine = 6144
elif utils.flt_comp(boxsize, 64.):
utils.print_msg('Setting conversion factors for 64/h Mpc box')
nbox_fine = 3456
elif utils.flt_comp(boxsize, 37.):
utils.print_msg('Setting conversion factors for 37/h Mpc box')
nbox_fine = 2048
else:
raise Exception('Invalid boxsize (%.3f cMpc)' % boxsize_cMpc)
M_box = const.rho_matter*(LB*const.Mpc)**3 # mass in box (g, not M0)
M_grid = M_box/(float(nbox_fine)**3)
lscale = (LB)/float(nbox_fine)*const.Mpc # size of a cell in cm, comoving
tscale = 2.0/(3.0*np.sqrt(const.Omega0)*const.H0/const.Mpc*1.e5) # time scale, when divided by (1+z)2
velconvert = lambda z: lscale/tscale*(1.0+z)/1.e5
def remove_client(self, client_conn, client_ip=None):
"""Remove client connection."""
length = len(self.client_conns)
data_in_this_edge = 0
number_of_client_in_this_edge = 0
temp = {}
for key, value in self.client_conns.items():
if value[0] != client_conn:
temp[key] = value
else:
data_in_this_edge = value[1]
number_of_client_in_this_edge = value[2]
self.client_conns = temp
if length == len(self.client_conns):
return
# critical section
with self._key_lock:
self.sum -= data_in_this_edge
self.n_client -= number_of_client_in_this_edge
self.n_element -= 1
if client_ip is not None:
print_msg("Client " + client_ip + " disconnected.")
def wait_for_clients(self):
"""Wait for clients' request for connection and provide worker thread
serving the client.
"""
while True:
# Wait for client
client_conn, (client_ip, client_port) = self.socket.accept()
# Reflect the wait is done
client_ip = client_ip + ":" + str(client_port)
if client_ip in self.client_conns:
self.client_conns[client_ip][0].close()
self.n_element -= 1
self.client_conns[client_ip] = []
self.client_conns[client_ip].append(client_conn)
# init the value of the client
self.client_conns[client_ip].append(0)
self.n_element += 1
print_msg(client_ip + " connected")
self.send_to_client(self.avg, client_conn, client_ip)
# Provide worker thread to serve client
worker_thread = threading.Thread(
target=self.handle_request,
args=(client_conn, client_ip)
)
worker_thread.daemon = True
worker_thread.start()
def process_packets(self, Q, file_path):
npack = 0
state = 0
n = 0
samples = []
start = time.time()
while (1):
tmp = Q.get()
samples.append(tmp)
#utils.print_msg(n, DEBUG)
packets = self.tnc.processBuffer(tmp)
self.decode_packets(packets)
for ax in packets:
npack = npack + 1
utils.print_msg("Number of Packets: " + str(npack), DEBUG)
#utils.print_msg((str(npack) +")", str(ax)), DEBUG)
if state == 0 and ax.destination[:5] == b"BEGIN":
#f1 = open(dir + "rec_"+ax.info,"wb")
start = time.time()
f1 = open(file_path + ".gz", "wb")
state = 1
elif state == 1 and ax.destination[:3] == b"END":
state = 2
f1.close()
break
elif state == 1:
f1.write(ax.info)
#print("write")
if state == 2:
break
n += 1
#if (time.time() - start > 75):
#break
print("Elapsed Time : " + str(time.time() - start))
return samples
def main():
args = init_args()
# unpack variables from args. allow flexible changes without using args
user = args.user
serial_number = args.s
file_path = args.f
jpeg_quality = args.q
downsample = args.d
test_number = args.test
type = args.type
if test_number != -1:
test.run_tests(user, serial_number)
else:
if args.type == "r": # receiver
utils.print_msg("Receiver!", DEBUG)
receiver_main(user, serial_number, file_path)
if args.type == "t": # transmitter
utils.print_msg("Transmitter!", DEBUG)
transmitter_main(user, serial_number, file_path, jpeg_quality,
downsample)
def execute(args):
persistence.connect_existing(os.getcwd())
last_trial_id = persistence.last_trial_id()
if not (1 <= args.trial[0] <= last_trial_id
and 1 <= args.trial[1] <= last_trial_id):
utils.print_msg('inexistent trial id', True)
sys.exit(1)
trial_before = persistence.load_trial(args.trial[0]).fetchone()
modules_before = persistence.load_dependencies()
environment_before = persistence.load('environment_attr',
['name', 'value'],
trial_id=trial_before['id'])
trial_after = persistence.load_trial(args.trial[1]).fetchone()
modules_after = persistence.load_dependencies()
environment_after = persistence.load('environment_attr', ['name, value'],
trial_id=trial_after['id'])
diff_trials(trial_before, trial_after)
if args.modules:
diff_modules(set(modules_before.fetchall()),
set(modules_after.fetchall()))
if args.environment:
diff_environment(set(environment_before.fetchall()),
set(environment_after.fetchall()))
def generate_packets(self, Qout, callsign, file_path):
utils.print_msg("Putting packets in Queue", DEBUG)
Qout.put("KEYON")
tmp = self.tnc.modulatePacket(callsign,
"",
"BEGIN",
file_path,
preflags=20,
postflags=2)
Qout.put(tmp)
Qout = self.packets_to_queue(file_path, callsign, Qout, 256)
tmp = self.tnc.modulatePacket(callsign,
"",
"END",
"This is the end of transmission",
preflags=2,
postflags=20)
Qout.put(tmp)
Qout.put("KEYOFF")
Qout.put("EOT")
utils.print_msg(
"Done generating packets. Generated {} packets".format(
Qout.qsize() - 3), DEBUG)
return Qout
def print_function_activations(function_activation):
utils.print_msg('this trial has the following function activation graph:',
True)
for inner_function_activation in persistence.load(
'function_activation', caller_id=function_activation['id']):
print_function_activation(inner_function_activation)
def execute(args):
utils.verbose = args.verbose
utils.print_msg('removing noWorkflow boilerplate')
args_script = args.script
args.script = os.path.realpath(args.script[0])
if not os.path.exists(args.script): # TODO: check this using argparse
utils.print_msg('the script does not exist', True)
sys.exit(1)
script_dir = os.path.dirname(args.script)
sys.path[0] = script_dir # Replace now's dir with script's dir in front of module search path.
# Clear argv
sys.argv = args_script
# Clear up the __main__ namespace
import __main__
__main__.__dict__.clear()
__main__.__dict__.update({'__name__' : '__main__',
'__file__' : args.script,
'__builtins__': __builtins__,
})
with open(args.script) as f:
metascript = {
'code': f.read(),
'path': args.script,
'compiled': None,
}
run(script_dir, args, metascript, __main__)
def setUp(self):
os.chdir(CURRENT_DIR)
self.test_name = os.path.basename(CURRENT_DIR)
self.test_info = load_test_info(self.test_name)
self.coll_name = self.test_info['coll_names'][0]
self.conn_src = self.test_info['conn_src']
self.conn_dest = self.test_info['conn_dest']
self.db_src = self.test_info['db_src']
self.db_dest = self.test_info['db_dest']
self.coll_src = self.db_src[self.coll_name]
self.coll_dest = self.db_dest[self.coll_name]
remove_res(self.test_info['temp_res'])
setup_dataset(
coll=self.coll_src,
dataset_file=os.path.join(CURRENT_DIR, 'data.json')
)
print_msg('Setting up progress file')
with open('current_progress.yaml', 'w') as output:
output.write("{}: 555317f7d290053143db668b\n".format(
self.coll_name
))
def communicate_with_server(self):
"""Generate & send a number to server & receive new input parameter."""
while True:
# Receive input parameter from server
data_rcv = pickle.loads(self.server.recv(1024))
print_msg("Received from server: " + str(data_rcv))
# Generate new number
# nntrain.main()
#Start training
if not self.trained:
type = "clients"
home_ip = self.server.getsockname()[0]
home_port = self.server.getsockname()[1]
home_address = str(home_ip) + ":" + str(home_port)
train.train_caller(1, 1, 0, 1, 0.01, type, home_address)
# Trained completed, send data to upper layers
print_msg("Ready to send file")
file_name = home_address + ".pt"
path = "trained/" + type + "/" + file_name
file_to_send = open(path, "rb")
data = file_to_send.read(CHUNKSIZE)
# data_to_send = pickle.dumps(data)
self.server.sendall(data)
def execute(args):
persistence.connect_existing(os.getcwd())
last_trial_id = persistence.last_trial_id()
trial_id = args.trial if args.trial != None else last_trial_id
if not 1 <= trial_id <= last_trial_id:
utils.print_msg('inexistent trial id', True)
sys.exit(1)
print_trial(persistence.load_trial(trial_id).fetchone())
if args.modules:
print_modules(persistence.load_dependencies())
if args.function_defs:
print_function_defs(persistence.load('function_def',
trial_id=trial_id))
if args.environment:
environment = {
attr['name']: attr['value']
for attr in persistence.load('environment_attr', trial_id=trial_id)
}
utils.print_map(
'this trial has been executed under the following environment conditions',
environment)
if args.function_activations:
print_function_activations(
persistence.load('function_activation',
caller_id=None,
trial_id=trial_id).fetchone())
if args.file_accesses:
print_file_accesses(persistence.load('file_access', trial_id=trial_id))
def tearDown(self):
print_msg("Dropping {} in source".format(
self.coll_name
))
self.coll_src.drop()
self.conn_src.close()
remove_res(self.test_info['temp_res'])
def set_up(self):
"""Set up connection with server"""
print_msg("Creating connection to server")
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.connect((self.server_ip, self.server_port))
print_msg("Connection established")
def test_decode_iss_packet(user, serial_number):
fs, sig = waveread("test_files/ISSpkt.wav")
r = receiver.Receiver(user, serial_number, fs=fs)
packets = r.parse_data(sig)
decode_packets(packets)
expected = 1
utils.print_msg("Found {} packet. Expected {}".format(found, expected), DEBUG)
r.terminate()
def print_modules(modules):
utils.print_msg('this trial depends on the following modules:', True)
output = []
for module in modules:
output.append(
' Name: {name}\n Version: {version}\n Path: {path}\n Code hash: {code_hash}'
.format(**module))
print('\n\n'.join(output))
def tearDown(self):
print_msg("Dropping {} in source and destination".format(self.coll_name))
self.coll_src.drop()
self.coll_dest.drop()
self.conn_src.close()
self.conn_dest.close()
remove_res(self.test_info["temp_res"])
def test_decode_mult_iss_packets(user, serial_number):
fs, sig = waveread("test_files/ISS.wav")
r = receiver.Receiver(user, serial_number, fs=fs)
packets = r.parse_data(sig)
decode_packets(packets)
found = len(filter(lambda ax: ax.info != 'bad packet', map(r.tnc.decodeAX25, filter(lambda pkt: len(pkt) > 200, packets))))
expected = 24
utils.print_msg("Found {} packet. Expected {}".format(found, expected), DEBUG)
r.terminate()
def tearDown(self):
print_msg("Dropping {} in source and destination".format(
self.coll_name))
self.coll_src.drop()
self.coll_dest.drop()
self.conn_src.close()
self.conn_dest.close()
remove_res(self.test_info['temp_res'])
def execute(args):
persistence.connect_existing(os.getcwd())
print_msg('trials available in the provenance store:', True)
for trial in persistence.load('trial'):
text = ' Trial {id}: {script} {arguments}'.format(**trial)
indent = text.index(': ') + 2
print(text)
print('{indent}with code hash {code_hash}'.format(indent = ' ' * indent, **trial))
print('{indent}ran from {start} to {finish}'.format(indent = ' ' * indent, **trial))
def shut_down(self):
"""Properly shut down server."""
print_msg("Shutting down socket in client")
data_to_send = pickle.dumps("close")
self.server.send(data_to_send)
self.server.shutdown(socket.SHUT_RDWR)
self.server.close()
def shut_down(self):
"""Properly shut down server."""
print_msg("Shutting down server.")
data_to_send = pickle.dumps('close')
self.socket.sendall(data_to_send)
self.socket.shutdown(socket.SHUT_RDWR)
self.socket.close()
def set_up_conn_to_upper_server(self):
"""Set up connection to server."""
print_msg("Creating connection to upper server")
self.upper_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.upper_server.connect(
(self.upper_server_ip, self.upper_server_port))
print_msg("Connection established")
def remove_client(self, client_conn, client_ip=None):
"""Remove client connection."""
try:
self.client_conns.remove(client_conn)
except ValueError:
return
if client_ip is not None:
print_msg("Client " + client_ip + " disconnected.")
def test_decode_iss_packet(user, serial_number):
fs, sig = waveread("test_files/ISSpkt.wav")
r = receiver.Receiver(user, serial_number, fs=fs)
packets = r.parse_data(sig)
decode_packets(packets)
expected = 1
utils.print_msg("Found {} packet. Expected {}".format(found, expected),
DEBUG)
r.terminate()
def broadcast_on_schedule(self):
"""Broadcast on schedule."""
start_time = time.time()
delay_time = 10.0
max_delay = 30.0
min_delay = 5.0
while True:
if len(self.client_conns) == 0:
start_time = time.time()
elif (time.time() - start_time > delay_time
or len(self.clients_responded) == len(self.client_conns)):
with self._key_lock:
if self.total_weight == 0:
delay_time = min([delay_time * 2.0, max_delay])
start_time = time.time()
continue
if len(self.clients_responded) == len(self.client_conns):
delay_time = max([delay_time / 2.0, min_delay])
else:
delay_time = min([delay_time * 1.1, max_delay])
start_time = time.time()
self.clients_responded = set()
self.model.fc.weight.data = self.sum / self.total_weight
self.sum = torch.zeros(self.model.fc.weight.shape)
self.total_weight = 0
self.seqnum = randint(0, 0xFFFFFF)
self.startSendTime = time.time()
self.broadcast_to_clients({
'avg':
self.model.fc.weight.data.clone(),
'seqnum':
self.seqnum
})
train_loss = 0.0
for i, (data, target) in enumerate(self.dl):
if i > 100:
break
output = self.model(data)
loss = self.criterion(output, target)
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
# train_loss += loss.item() / len(self.dl)
train_loss += loss.item() / 100
print_msg("Current loss value: " + str(train_loss))
print_msg("------------------------------------")
time.sleep(0.1)
def bak():
for section in config.sections():
bakdir = config.get(section, "bakdir")
bakdir_fullpath = os.path.join(root, bakdir)
action = config.get(section, "action")
target = config.get(section, "target")
custom_cmd = config.get(section, "cmd", None)
utils.print_msg("> Excuting {}...".format(section))
cmd = CommandHandler(action, bakdir_fullpath, target, custom_cmd)
cmd.execute()
def execute(args):
persistence.connect_existing(os.getcwd())
last_trial_id = persistence.last_trial_id()
trial_id = args.trial if args.trial != None else last_trial_id
if not 1 <= trial_id <= last_trial_id:
utils.print_msg('inexistent trial id', True)
sys.exit(1)
print(export_facts(trial_id))
if args.rules:
print(export_rules(trial_id))
def set_up(self):
"""Set up socket."""
print_msg("Starting server.")
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.socket.bind(('', self.port))
self.socket.listen(100)
print_msg("Server started.")
def execute(args):
persistence.connect_existing(os.getcwd())
last_trial_id = persistence.last_trial_id()
trial_id = args.trial if args.trial != None else last_trial_id
if not 1 <= trial_id <= last_trial_id:
utils.print_msg('inexistent trial id', True)
sys.exit(1)
print(export_facts(trial_id))
if args.rules:
print(export_rules(trial_id))
def find_diff(dir1, dir2):
# Eg.: $ gfind dir1 dir2 -printf "%P\t%s\n" | sort | uniq -u
cmd = '{} "{}" "{}" -printf "%P\\t%s\\n" | {} | {} -u'.format(
FIND_CMD, dir1, dir2, SORT_CMD, UNIQ_CMD)
output = subprocess.check_output(cmd, shell=True).rstrip()
if not output:
utils.print_msg('No diff\n')
for line in output.splitlines():
if line:
relative_path, size = line.split('\t')
_print_diff(relative_path, size, dir1, dir2)
def req_default(ip_addr):
try:
ret_code = requests.get("http://" + ip_addr,
verify=False,
timeout=30).status_code
if ret_code != 200 and ret_code != 302:
return False
else:
return True
except ConnectionError, e:
print_msg(e)
return False
def _group_by_size_and_checksum(dupes_map):
total_num = 0
for dupes in dupes_map.values():
total_num += len(dupes)
i = 1
for size, dupes in dupes_map.items():
utils.print_msg(
'\n> [{}/{}] Comparing checksums for files with the same size:\n{}'
.format(i, total_num, '\n'.join(dupes)))
i += len(dupes)
actual_dupes = _select_dupes_with_same_checksum(dupes, total_num)
dupes_map[size] = actual_dupes
def test_image(user, serial_number):
"""
loop back mode
"""
# setup variables for tests
callsign = "KK6MRI"
fname = "calBlue.tiff"
dir = "images/"
f = open(dir + fname,"rb")
fs = 48000
Abuffer = 1024
Nchunks = 12
# prepare transmitter and receiver
t = transmitter.Transmitter(user, serial_number, fs=fs, Abuffer=Abuffer, Nchunks=Nchunks)
r = receiver.Receiver(user, serial_number, fs=fs, Abuffer=Abuffer, Nchunks=Nchunks)
# prepare i/o queues
Qin = Queue.Queue()
Qout = Queue.Queue()
# create a control fifo to kill threads when done
cQin = Queue.Queue()
cQout = Queue.Queue()
# create a pyaudio object
p = pyaudio.PyAudio()
# initialize a recording thread.
t_rec = threading.Thread(target = aprs.record_audio, args = (Qin, cQin, p, fs, r.dusb_in))
t_play = threading.Thread(target = aprs.play_audio, args = (Qout, cQout, p, fs, t.dusb_out))
# generate packets and put into output queue
Qout = t.generate_packets(Qout, callsign)
# start the recording and playing threads
t_rec.start()
time.sleep(2)
t_play.start()
starttime = time.time()
# process packets and put them into input queue
r.process_packets(Qin, file_path)
utils.print_msg(time.time() - starttime, DEBUG)
cQout.put("EOT")
cQin.put("EOT")
t.terminate()
r.terminate()
def decodeAX25(self,bits, deepsearch=False):
ax = ax25.AX25()
ax.info = "bad packet"
bitsu = ax25.bit_unstuff(bits[8:-8])
foundPacket = False
if (self.genfcs(bitsu[:-16]).tobytes() == bitsu[-16:].tobytes()):
foundPacket = True
elif deepsearch:
tbits = bits[8:-8]
for n in range(0,len(tbits)):
tbits[n] = not tbits[n]
if (self.genfcs(bitsu[:-16]).tobytes() == bitsu[-16:].tobytes()):
foundPacket = True
print("Success deep search")
break
tbits[n] = not tbits[n]
if foundPacket == False:
return ax
bytes = bitsu.tobytes()
ax.destination = ax.callsign_decode(bitsu[:56])
source = ax.callsign_decode(bitsu[56:112])
source = str(source)
if source[-1].isdigit() and source[-1]!="0":
ax.source = b"".join((source[:-1],'-',source[-1]))
else:
ax.source = source[:-1]
digilen=0
if bytes[14]=='\x03' and bytes[15]=='\xf0':
digilen = 0
else:
for n in range(14,len(bytes)-1):
if bytes[n] & 1:
digilen = (n-14)+1
break
utils.print_msg("Digilen = " + str(digilen), DEBUG)
# if digilen > 56:
# return ax
ax.digipeaters = ax.callsign_decode(bitsu[112:112+digilen*8])
ax.info = bitsu[112+digilen*8+16:-16].tobytes()
return ax
def generate_packets(self, Qout, callsign, file_path):
utils.print_msg("Putting packets in Queue", DEBUG)
Qout.put("KEYON")
tmp = self.tnc.modulatePacket(callsign, "", "BEGIN", file_path , preflags=20, postflags=2 )
Qout.put(tmp)
Qout = self.packets_to_queue(file_path, callsign, Qout, 256)
tmp = self.tnc.modulatePacket(callsign, "", "END", "This is the end of transmission", preflags=2, postflags=20)
Qout.put(tmp)
Qout.put("KEYOFF")
Qout.put("EOT")
utils.print_msg("Done generating packets. Generated {} packets".format(Qout.qsize()-3), DEBUG)
return Qout
def transmit_file(self, file_path, callsign="KM6BHD"):
Qout = Queue.Queue()
cQout = Queue.Queue()
p = pyaudio.PyAudio()
t_play = threading.Thread(target = aprs.play_audio , args=(Qout, cQout, p, self.tnc.fs, self.dusb_out, self.s))
Qout = self.generate_packets(Qout, callsign, file_path)
utils.print_msg("Playing packets", DEBUG)
t_play.start()
time.sleep(75)
cQout.put("EOT")
time.sleep(3)
p.terminate()
utils.print_msg("Terminating", DEBUG)
def connect(path):
global content_path, db_conn
provenance_path = os.path.join(path, PROVENANCE_DIRNAME)
content_path = os.path.join(provenance_path, CONTENT_DIRNAME)
if not os.path.isdir(content_path):
os.makedirs(content_path)
db_path = os.path.join(provenance_path, DB_FILENAME)
new_db = not os.path.exists(db_path)
db_conn = sqlite3.connect(db_path)
db_conn.row_factory = sqlite3.Row
if new_db:
print_msg('creating provenance database')
# Accessing the content of a file via setuptools
with db_conn as db:
db.executescript(resource_string(__name__, DB_SCRIPT))
def test_freshrun(self):
self.coll_dest.drop()
print_msg("Running {} test".format(self.test_name))
os.system(
"../../src/mongob --config config.yaml" + " --progress-file current_progress.yaml" + " --log backup.log"
)
print_msg("Checking result for {}".format(self.test_name))
with open(self.test_info["dataset_file"], "r") as input:
data_from_file = json_loads(input.read())
data_from_file.sort(key=lambda x: x["_id"])
data_from_src = list(self.coll_src.find().sort("_id", 1))
data_from_dest = list(self.coll_dest.find().sort("_id", 1))
self.assertEqual(data_from_file, data_from_src)
self.assertEqual(data_from_file, data_from_dest)
def test_incremental_objectid(self):
self.coll_dest.drop()
print_msg('Running {} test'.format(self.test_name))
os.system(
'../../src/mongob --config config.yaml'
+ ' --progress-file current_progress.yaml'
+ ' --log backup.log'
)
print_msg('Checking result for {}'.format(self.test_name))
with open(self.test_info['dataset_file'], 'r') as input:
data_from_file = json_loads(input.read())
data_from_file.sort(key=lambda x: x['_id'])
data_from_src = list(self.coll_src.find().sort('_id', 1))
data_from_dest = list(self.coll_dest.find().sort('_id', 1))
self.assertEqual(data_from_file, data_from_src)
self.assertEqual(data_from_file[4:], data_from_dest)
def execute(args):
persistence.connect_existing(os.getcwd())
last_trial_id = persistence.last_trial_id()
if not (1 <= args.trial[0] <= last_trial_id and 1 <= args.trial[1] <= last_trial_id):
utils.print_msg('inexistent trial id', True)
sys.exit(1)
trial_before = persistence.load_trial(args.trial[0]).fetchone()
modules_before = persistence.load_dependencies()
environment_before = persistence.load('environment_attr', ['name', 'value'], trial_id = trial_before['id'])
trial_after = persistence.load_trial(args.trial[1]).fetchone()
modules_after = persistence.load_dependencies()
environment_after = persistence.load('environment_attr', ['name, value'], trial_id = trial_after['id'])
diff_trials(trial_before, trial_after)
if args.modules:
diff_modules(set(modules_before.fetchall()), set(modules_after.fetchall()))
if args.environment:
diff_environment(set(environment_before.fetchall()), set(environment_after.fetchall()))
def collect_provenance(args, metascript):
print_msg(' registering environment attributes')
environment = collect_environment_provenance()
persistence.store_environment(environment)
if args.bypass_modules:
print_msg(' using previously detected module dependencies (--bypass-modules).')
else:
print_msg(' searching for module dependencies')
finder = modulefinder.ModuleFinder()
finder.run_script(metascript['path'])
print_msg(' registering provenance from {} modules'.format(len(finder.modules) - 1))
modules = collect_modules_provenance(finder.modules)
persistence.store_dependencies(modules)
def execute(args):
persistence.connect_existing(os.getcwd())
last_trial_id = persistence.last_trial_id()
trial_id = args.trial if args.trial != None else last_trial_id
if not 1 <= trial_id <= last_trial_id:
utils.print_msg('inexistent trial id', True)
sys.exit(1)
last_trial = persistence.load_trial(last_trial_id).fetchone()
trial = persistence.load_trial(trial_id).fetchone()
if os.path.isfile(trial['script']):
with open(trial['script'], 'rb') as f:
code_hash = persistence.put(f.read())
if code_hash != last_trial['code_hash']:
# ToDo: create trial
pass
load_file = persistence.get(trial['code_hash'])
with open(trial['script'], 'w') as f:
f.write(load_file)
utils.print_msg('File {} from trial {} restored'.format(
trial['script'], trial_id), True)
def main():
args = init_args()
# unpack variables from args. allow flexible changes without using args
user = args.user
serial_number = args.s
file_path = args.f
jpeg_quality = args.q
downsample = args.d
test_number = args.test
type = args.type
if test_number != -1:
test.run_tests(user, serial_number)
else:
if args.type == "r": # receiver
utils.print_msg("Receiver!", DEBUG)
receiver_main(user, serial_number, file_path)
if args.type == "t": # transmitter
utils.print_msg("Transmitter!", DEBUG)
transmitter_main(user, serial_number, file_path, jpeg_quality, downsample)
def diff_environment(before, after):
(added, removed, replaced) = diff_set(before, after)
utils.print_msg('{} environment attributes added:'.format(len(added),), True)
utils.print_environment_attrs(added)
print()
utils.print_msg('{} environment attributes removed:'.format(len(removed),), True)
utils.print_environment_attrs(removed)
print()
utils.print_msg('{} environment attributes replaced:'.format(len(replaced),), True)
for (attr_removed, attr_added) in replaced:
print(' Environment attribute {} changed from {} to {}'.format(attr_removed['name'], attr_removed['value'], attr_added['value']))
def diff_modules(before, after):
(added, removed, replaced) = diff_set(before, after)
utils.print_msg('{} modules added:'.format(len(added),), True)
utils.print_modules(added)
print()
utils.print_msg('{} modules removed:'.format(len(removed),), True)
utils.print_modules(removed)
print()
utils.print_msg('{} modules replaced:'.format(len(replaced),), True)
for (module_removed, module_added) in replaced:
print(' Name: {}'.format(module_removed['name'],))
diff_dict(module_removed, module_added)
print()
def test_sms(user, serial_number):
utils.print_msg("Running SMS Test", DEBUG)
t = transmitter.Transmitter(user, serial_number, baud=1200, space_f=2200)
callsign = "KM6BHD"
Digi =b'WIDE1-1,WIDE2-1'
dest = "APCAL"
# Uncomment to Send Email
#info = ":EMAIL :[email protected] Hi, test email!"
# Uncomment to Send an SMS message to a phone number
#info = ":SMSGTE :@4089312267 Testing from lab computers"
#uncomment to show yourself on mt everest
#info = "=2759.16N/08655.30E[I'm on the top of the world"
#uncomment to send to everyone on the APRS system near you
#info = ":ALL : CQCQCQ I would like to talk to you!"
#uncomment to report position
info = "=3752.50N/12215.43WlIm using a laptop in Cory Hall!"
#uncomment to send a status message
#info = ">I like radios"
utils.print_msg("Preparing packet...", DEBUG)
packet = ax25.UI(
destination=dest,
source=callsign,
info=info,
digipeaters=Digi.split(b','),
)
utils.print_msg("Transmitting packet...", DEBUG)
t.transmit_packet(packet)
time.sleep(2)
utils.print_msg("Finished transmitting packet...", DEBUG)
t.terminate()
def diff_trials(before, after):
utils.print_msg('trial diff:', True)
diff_dict(before, after)
print()
def connect_existing(path):
if not has_provenance(path):
print_msg('there is no provenance store in the current directory',
True)
sys.exit(1)
connect(path)
import stats
import parser as myparser
import analyzer
parser = classify_symms_args()
# parse command line arguments
args = parser.parse_args()
problems = []
if not args.symmfolder is None:
problems = list_files(args.symmfolder, '.symm')
else:
parser.error('You must provide a folder to process.')
print_msg('===========================================')
print_msg(('Starting classify_symms.py'))
print_msg('===========================================')
print_msg(('Variable File: %s' % args.varfile ))
print_msg(('Input Folder: %s ' % args.symmfolder))
print_msg(('Output Folder: %s' % args.folder))
print_msg(('Human Readable: %s' % (not args.computer)))
print_msg('-------------------------------------------',args.silent)
print_msg(('%s files were find' % len(problems)),args.silent)
print_msg(('Loading variables file: %s' % args.varfile),args.silent)
vars_t = analyzer.load_vars_file(args.varfile)
stats_list = []
for p in problems:
try:
def run(script_dir, args, metascript, __main__):
utils.print_msg('setting up local provenance store')
persistence.connect(script_dir)
utils.print_msg('collecting definition provenance')
prov_definition.collect_provenance(args, metascript)
utils.print_msg('collecting deployment provenance')
prov_deployment.collect_provenance(args, metascript)
utils.print_msg('collection execution provenance')
prov_execution.enable(args, metascript)
utils.print_msg(' executing the script')
try:
if metascript['compiled'] is None:
metascript['compiled'] = compile(
metascript['code'], metascript['path'], 'exec')
exec(metascript['compiled'], __main__.__dict__)
except SystemExit as ex:
prov_execution.disable()
utils.print_msg('the execution exited via sys.exit(). Exit status: {}'.format(ex.code), ex.code > 0)
except Exception as e:
prov_execution.disable()
print e
utils.print_msg('the execution finished with an uncaught exception. {}'.format(traceback.format_exc()), True)
else:
prov_execution.disable()
prov_execution.store() # TODO: exceptions should be registered as return from the activation and stored in the database. We are currently ignoring all the activation tree when exceptions are raised.
utils.print_msg('the execution of trial {} finished successfully'.format(persistence.trial_id))
return prov_execution.provider
def processBuffer(self, buff_in):
# function processes an audio buffer. It collect several small into a large one
# Then it demodulates and finds packets.
#
# The function operates as overlapp and save
# The function returns packets when they become available. Otherwise, returns empty list
N = self.N
NN = N*3-3
Nchunks = self.Nchunks
Abuffer = self.Abuffer
fs = self.fs
Ns = self.Ns
validPackets=[]
packets=[]
NRZI=[]
idx = []
bits = []
# Fill in buffer at the right plave
self.buff[NN+self.chunk_count*Abuffer:NN+(self.chunk_count+1)*Abuffer] = buff_in.copy()
self.chunk_count = self.chunk_count + 1
# number of chunk reached -- process large buffer
if self.chunk_count == Nchunks:
# Demodulate to get NRZI
NRZI = self.demod(self.buff)
# compute sampling points, using PLL
idx = self.PLL(NRZI)
# Sample and make a decision based on threshold
bits = bitarray.bitarray((NRZI[idx]>0).tolist())
# In case that buffer is too small raise an error -- must have at least 7 bits worth
if len(bits) < 7:
raise ValueError('number of bits too small for buffer')
# concatenate end of previous buffer to current one
bits = self.oldbits + self.NRZI2NRZ(bits)
# store end of bit buffer to next buffer
self.oldbits = bits[-7:].copy()
# look for packets
packets = self.findPackets(bits)
utils.print_msg("Found " + str(len(packets)) + " packets", DEBUG)
# Copy end of sample buffer to the beginning of the next (overlapp and save)
self.buff[:NN] = self.buff[-NN:].copy()
# reset chunk counter
self.chunk_count = 0
# checksum test for all detected packets
for n in range(0,len(packets)):
if len(packets[n]) > 200:
ax = self.decodeAX25(packets[n])
#print (" |DEST:" + ax.destination[:-1].decode('ascii') + " |SRC:" + " |DIGI:" + " |", ax.info, "|")
if ax.info != 'bad packet':
validPackets.append(ax)
return validPackets
