def test_flter_bloom(self):
""" Tests that the result of multiple insertions of encoded()'s is a
superset of the encoded cells, for multiple random sets """
nymseeds = {}
for i in range(30):
nymseeds[i] = []
for _ in range(30):
nymseeds[i].append(random.randint(0, 9))
d = RequestDecoder(nymseeds)
encs = [RequestEncoder(nymseeds[i]) for i in nymseeds]
random.seed()
for _ in range(30):
samplesize = random.randint(1, len(nymseeds))
nyms = sorted(random.sample(range(len(nymseeds)), samplesize))
these_encs = [encs[i] for i in nyms]
cell = Bits(uint=0, length=d.cell_bit_length)
for enc in these_encs:
cell |= Bits(enc.encode())
self.assertEqual(d.full | cell, d.full)
self.assertTrue(d.check(cell.tobytes()))
print("DONE CHECKING NOW DECODING")
decoded = sorted(d.decode(cell.tobytes()))
print("DONE CHECKING NOW VERIFYING")
for elt in nyms:
self.assertTrue(elt in decoded,
msg="elt: {0} nyms: {1} decoded: {2}\n"
.format(elt, nyms, decoded) + \
"encs: {0}\n cell: {1}"
.format([i.encode() for i in encs], cell))
def uncompress_golomb_coding(coded_bytes, hash_length, M):
"""Given a bytstream produced using golomb_coded_bytes, uncompress it."""
ret_list = []
instream = BitStream(bytes=coded_bytes, length=len(coded_bytes) * 8)
hash_len_bits = hash_length * 8
m_bits = int(math.log(M, 2))
# First item is a full hash value.
prev = instream.read("bits:%d" % hash_len_bits)
ret_list.append(prev.tobytes())
while (instream.bitpos + m_bits) <= instream.length:
# Read Unary-encoded value.
read_prefix = 0
curr_bit = instream.read("uint:1")
while curr_bit == 1:
read_prefix += 1
curr_bit = instream.read("uint:1")
assert curr_bit == 0
# Read r, assuming M bits were used to represent it.
r = instream.read("uint:%d" % m_bits)
curr_diff = read_prefix * M + r
curr_value_int = prev.uint + curr_diff
curr_value = Bits(uint=curr_value_int, length=hash_len_bits)
ret_list.append(curr_value.tobytes())
prev = curr_value
return ret_list
def uncompress_golomb_coding(coded_bytes, hash_length, M):
"""Given a bytstream produced using golomb_coded_bytes, uncompress it."""
ret_list = []
instream = BitStream(
bytes=coded_bytes, length=len(coded_bytes) * 8)
hash_len_bits = hash_length * 8
m_bits = int(math.log(M, 2))
# First item is a full hash value.
prev = instream.read("bits:%d" % hash_len_bits)
ret_list.append(prev.tobytes())
while (instream.bitpos + m_bits) <= instream.length:
# Read Unary-encoded value.
read_prefix = 0
curr_bit = instream.read("uint:1")
while curr_bit == 1:
read_prefix += 1
curr_bit = instream.read("uint:1")
assert curr_bit == 0
# Read r, assuming M bits were used to represent it.
r = instream.read("uint:%d" % m_bits)
curr_diff = read_prefix * M + r
curr_value_int = prev.uint + curr_diff
curr_value = Bits(uint=curr_value_int, length=hash_len_bits)
ret_list.append(curr_value.tobytes())
prev = curr_value
return ret_list
def save_bits_to_file(filepath, bits):
# get file extension
bitstring = Bits(bin=bits)
mime = Magic(mime=True)
mime_type = mime.from_buffer(bitstring.tobytes())
with open(f"{filepath}/file{mimetypes.guess_extension(type=mime_type)}",
"wb") as f:
bitstring.tofile(f)
def save_bits_to_file(file_path, bits):
# get file extension
bitstring = Bits(bin=bits)
mime = Magic(mime=True)
mime_type = mime.from_buffer(bitstring.tobytes())
# If filepath not passed in use defualt
# otherwise used passed in filepath
if file_path == None:
filepath = f"file{mimetypes.guess_extension(type=mime_type)}"
else:
filepath = file_path
with open(filepath, "wb") as f:
bitstring.tofile(f)
def datfile(self, _file):
""" _file should be the full path to the data file """
self.tickGroups[0] = tickGroup()
self.tickGroups[1] = tickGroup()
self.results = AnalyzeDatResults()
self.file = open(_file, 'rb')
## create a bits object with the content of the file
dataAsBits = Bits(self.file)
## 8 bits per byte
self.fileLength = (dataAsBits.length / 8)
self.memory = dataAsBits.tobytes()
try:
if self.getByte(128) != 0x55:
self.alternateStructure = True
except FileEnd:
self.close()
raise NotDatFile()
buildStr = self.getString(16)
if buildStr.find("BUILD") < 0:
self.close()
raise NotDatFile()
def uncompress_golomb_coding(coded_bytes, hash_length, M):
ret_list = []
instream = BitStream(
bytes=coded_bytes, length=len(coded_bytes) * hash_length)
hash_len_bits = hash_length * 8
m_bits = int(math.log(M, 2))
prev = instream.read("bits:%d" % hash_len_bits)
ret_list.append(prev.tobytes())
while instream.bitpos < instream.length:
read_prefix = 0
curr_bit = instream.read("uint:1")
while curr_bit == 1:
read_prefix += 1
curr_bit = instream.read("uint:1")
assert curr_bit == 0
r = instream.read("uint:%d" % m_bits)
curr_diff = read_prefix * M + r
curr_value_int = prev.uint + curr_diff
curr_value = Bits(uint=curr_value_int, length=hash_len_bits)
ret_list.append(curr_value.tobytes())
prev = curr_value
return ret_list
def multiple_exon_alnmt(gene_list, db_info):
print "process pid: %d, length of gene list: %d" % ( get_process_id(), len(gene_list))
[local_db, ensembl_db_name] = db_info
db = connect_to_mysql()
cfg = ConfigurationReader()
acg = AlignmentCommandGenerator()
cursor = db.cursor()
# find db ids adn common names for each species db
[all_species, ensembl_db_name] = get_species (cursor)
species = 'homo_sapiens'
switch_to_db (cursor, ensembl_db_name[species])
gene_ids = get_gene_ids (cursor, biotype='protein_coding', is_known=1)
# for each human gene
gene_ct = 0
tot = 0
ok = 0
no_maps = 0
no_pepseq = 0
no_orthologues = 0
min_similarity = cfg.get_value('min_accptbl_exon_sim')
#gene_list.reverse()
for gene_id in gene_list:
start = time()
gene_ct += 1
if not gene_ct%10: print gene_ct, "genes out of", len(gene_list)
switch_to_db (cursor, ensembl_db_name['homo_sapiens'])
print gene_ct, len(gene_ids), gene_id, gene2stable(cursor, gene_id), get_description (cursor, gene_id)
human_exons = filter (lambda e: e.is_known==1 and e.is_coding and e.covering_exon<0, gene2exon_list(cursor, gene_id))
human_exons.sort(key=lambda exon: exon.start_in_gene)
##################################################################
for human_exon in human_exons:
tot += 1
# find all orthologous exons the human exon maps to
maps = get_maps(cursor, ensembl_db_name, human_exon.exon_id, human_exon.is_known)
if verbose:
print "\texon no.", tot, " id", human_exon.exon_id,
if not maps:
print " no maps"
print human_exon
print
if not maps:
no_maps += 1
continue
# human sequence to fasta:
seqname = "{0}:{1}:{2}".format('homo_sapiens', human_exon.exon_id, human_exon.is_known)
switch_to_db (cursor, ensembl_db_name['homo_sapiens'])
[exon_seq_id, pepseq, pepseq_transl_start, pepseq_transl_end,
left_flank, right_flank, dna_seq] = get_exon_seqs (cursor, human_exon.exon_id, human_exon.is_known)
if (not pepseq):
if verbose and human_exon.is_coding and human_exon.covering_exon <0: # this should be a master exon
print "no pep seq for", human_exon.exon_id, "coding ", human_exon.is_coding,
print "canonical: ", human_exon.is_canonical
print "length of dna ", len(dna_seq)
no_pepseq += 1
continue
# collect seq from all maps, and output them in fasta format
hassw = False
headers = []
sequences = {}
exons_per_species = {}
for map in maps:
switch_to_db (cursor, ensembl_db_name[map.species_2])
if map.similarity < min_similarity: continue
exon = map2exon(cursor, ensembl_db_name, map)
pepseq = get_exon_pepseq (cursor,exon)
if (not pepseq):
continue
if map.source == 'sw_sharp':
exon_known_code = 2
hassw = True
elif map.source == 'usearch':
exon_known_code = 3
hassw = True
else:
exon_known_code = map.exon_known_2
seqname = "{0}:{1}:{2}".format(map.species_2, map.exon_id_2, exon_known_code)
headers.append(seqname)
sequences[seqname] = pepseq
# for split exon concatenation (see below)
if not map.species_2 in exons_per_species.keys():
exons_per_species[map.species_2] = []
exons_per_species[map.species_2].append ([ map.exon_id_2, exon_known_code]);
if (len(headers) <=1 ):
if verbose: print "single species in the alignment"
no_orthologues += 1
continue
# concatenate exons from the same gene - the alignment program might go wrong otherwise
concatenated = concatenate_exons (cursor, ensembl_db_name, sequences, exons_per_species)
fasta_fnm = "{0}/{1}.fa".format( cfg.dir_path['scratch'], human_exon.exon_id)
output_fasta (fasta_fnm, sequences.keys(), sequences)
# align
afa_fnm = "{0}/{1}.afa".format( cfg.dir_path['scratch'], human_exon.exon_id)
mafftcmd = acg.generate_mafft_command (fasta_fnm, afa_fnm)
ret = commands.getoutput(mafftcmd)
if (verbose): print 'almt to', afa_fnm
# read in the alignment
inf = erropen(afa_fnm, "r")
aligned_seqs = {}
for record in SeqIO.parse(inf, "fasta"):
aligned_seqs[record.id] = str(record.seq)
inf.close()
# split back the concatenated exons
if concatenated: split_concatenated_exons (aligned_seqs, concatenated)
human_seq_seen = False
for seq_name, sequence in aligned_seqs.iteritems():
# if this is one of the concatenated seqs, split them back to two
### store the alignment as bitstring
# Generate the bitmap
bs = Bits(bin='0b' + re.sub("[^0]","1", sequence.replace('-','0')))
# The returned value of tobytes() will be padded at the end
# with between zero and seven 0 bits to make it byte aligned.
# I will end up with something that looks like extra alignment gaps, that I'll have to return
msa_bitmap = bs.tobytes()
# Retrieve information on the cognate
cognate_species, cognate_exon_id, cognate_exon_known = seq_name.split(':')
if cognate_exon_known == '2':
source = 'sw_sharp'
elif cognate_exon_known == '3':
source = 'usearch'
else:
source = 'ensembl'
if (cognate_species == 'homo_sapiens'):
human_seq_seen = True
cognate_genome_db_id = species2genome_db_id(cursor, cognate_species) # moves the cursor
switch_to_db(cursor, ensembl_db_name['homo_sapiens']) # so move it back to h**o sapiens
# Write the bitmap to the database
#if (cognate_species == 'homo_sapiens'):
if verbose: # and (source=='sw_sharp' or source=='usearch'):
print "storing"
print human_exon.exon_id, human_exon.is_known
print cognate_species, cognate_genome_db_id, cognate_exon_id, cognate_exon_known, source
print sequence
if not msa_bitmap:
print "no msa_bitmap"
continue
store_or_update(cursor, "exon_map", {"cognate_genome_db_id":cognate_genome_db_id,
"cognate_exon_id":cognate_exon_id ,"cognate_exon_known" :cognate_exon_known,
"source": source, "exon_id" :human_exon.exon_id, "exon_known":human_exon.is_known},
{"msa_bitstring":MySQLdb.escape_string(msa_bitmap)})
ok += 1
commands.getoutput("rm "+afa_fnm+" "+fasta_fnm)
if verbose: print " time: %8.3f\n" % (time()-start);
print "tot: ", tot, "ok: ", ok
print "no maps ", no_pepseq
print "no pepseq ", no_pepseq
print "no orthologues ", no_orthologues
print
def multiple_exon_alnmt(species_list, db_info):
[local_db, ensembl_db_name] = db_info
verbose = False
db = connect_to_mysql()
cfg = ConfigurationReader()
acg = AlignmentCommandGenerator()
cursor = db.cursor()
for species in species_list:
print
print "############################"
print species
switch_to_db (cursor, ensembl_db_name[species])
gene_ids = get_gene_ids (cursor, biotype='protein_coding')
#gene_ids = get_theme_ids(cursor, cfg, 'wnt_pathway')
if not gene_ids:
print "no gene_ids"
continue
gene_ct = 0
tot = 0
ok = 0
no_maps = 0
no_pepseq = 0
no_paralogues = 0
for gene_id in gene_ids:
if verbose: start = time()
gene_ct += 1
if not gene_ct%100: print species, gene_ct, "genes out of", len(gene_ids)
if verbose:
print
print gene_id, gene2stable(cursor, gene_id), get_description (cursor, gene_id)
# get the paralogues - only the representative for the family will have this
paralogues = get_paras (cursor, gene_id)
if not paralogues:
if verbose: print "\t not a template or no paralogues"
continue
if verbose: print "paralogues: ", paralogues
# get _all_ exons
template_exons = gene2exon_list(cursor, gene_id)
if (not template_exons):
if verbose: print 'no exons for ', gene_id
continue
# find all template exons we are tracking in the database
for template_exon in template_exons:
if verbose: print template_exon.exon_id
maps = get_maps(cursor, ensembl_db_name, template_exon.exon_id,
template_exon.is_known, species=species, table='para_exon_map')
if not maps:
no_maps += 1
continue
# output to fasta:
seqname = "{0}:{1}:{2}".format('template', template_exon.exon_id, template_exon.is_known)
exon_seqs_info = get_exon_seqs (cursor, template_exon.exon_id, template_exon.is_known)
if not exon_seqs_info: continue
[exon_seq_id, pepseq, pepseq_transl_start, pepseq_transl_end,
left_flank, right_flank, dna_seq] = exon_seqs_info
if (not pepseq):
if ( template_exon.is_coding and template_exon.covering_exon <0): # this should be a master exon
print "no pep seq for", template_exon.exon_id, "coding ", template_exon.is_coding,
print "canonical: ", template_exon.is_canonical
print "length of dna ", len(dna_seq)
no_pepseq += 1
continue
tot += 1
sequences = {seqname:pepseq}
headers = [seqname]
for map in maps:
exon = map2exon(cursor, ensembl_db_name, map, paralogue=True)
pepseq = get_exon_pepseq (cursor,exon)
if (not pepseq):
continue
seqname = "{0}:{1}:{2}".format('para', map.exon_id_2, map.exon_known_2)
headers.append(seqname)
sequences[seqname] = pepseq
fasta_fnm = "{0}/{1}_{2}_{3}.fa".format( cfg.dir_path['scratch'], species, template_exon.exon_id, template_exon.is_known)
output_fasta (fasta_fnm, headers, sequences)
if (len(headers) <=1 ):
print "single species in the alignment (?)"
no_paralogues += 1
continue
# align
afa_fnm = "{0}/{1}_{2}_{3}.afa".format( cfg.dir_path['scratch'], species, template_exon.exon_id, template_exon.is_known)
mafftcmd = acg.generate_mafft_command (fasta_fnm, afa_fnm)
ret = commands.getoutput(mafftcmd)
# read in the alignment
inf = erropen(afa_fnm, "r")
if not inf:
print gene_id
continue
template_seq_seen = False
for record in SeqIO.parse(inf, "fasta"):
### store the alignment as bitstring
# Generate the bitmap
bs = Bits(bin='0b' + re.sub("[^0]","1", str(record.seq).replace('-','0')))
msa_bitmap = bs.tobytes()
# Retrieve information on the cognate
label, cognate_exon_id, cognate_exon_known = record.id.split(':')
if (label == 'template'):
template_seq_seen = True
# Write the bitmap to the database
#print "updating: ", template_exon.exon_id
store_or_update(cursor, "para_exon_map", {"cognate_exon_id" :cognate_exon_id,
"cognate_exon_known" :cognate_exon_known,
"exon_id" :template_exon.exon_id,
"exon_known" :template_exon.is_known},
{"msa_bitstring":MySQLdb.escape_string(msa_bitmap)})
inf.close()
ok += 1
commands.getoutput("rm "+afa_fnm+" "+fasta_fnm)
if verbose: print " time: %8.3f\n" % (time()-start);
outstr = species + " done \n"
outstr += "tot: %d ok: %d \n" % (tot, ok)
outstr += "no maps %d \n" % no_pepseq
outstr += "no pepseq %d \n" % no_pepseq
outstr += "no paralogues %d \n" % no_paralogues
outstr += "\n"
print outstr
# Requires bitstring module
# $ pip install bitstring
import socket
from time import sleep
from bitstring import Bits
UDP_ADDR = "10.0.0.9"
UDP_PORT = 5040
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
while True:
for servo_value in range(150,600):
bits = Bits('uint:16=' + str(servo_value))
print bits.bin
s.sendto(bits.tobytes(), (UDP_ADDR, UDP_PORT))
sleep(0.1)
def multiple_exon_alnmt(species_list, db_info):
[local_db, ensembl_db_name] = db_info
verbose = False
db = connect_to_mysql()
cfg = ConfigurationReader()
acg = AlignmentCommandGenerator()
cursor = db.cursor()
for species in species_list:
print()
print("############################")
print(species)
switch_to_db (cursor, ensembl_db_name[species])
gene_ids = get_gene_ids (cursor, biotype='protein_coding')
#gene_ids = get_theme_ids(cursor, cfg, 'wnt_pathway')
if not gene_ids:
print("no gene_ids")
continue
gene_ct = 0
tot = 0
ok = 0
no_maps = 0
no_pepseq = 0
no_paralogues = 0
for gene_id in gene_ids:
if verbose: start = time()
gene_ct += 1
if not gene_ct%100: print(species, gene_ct, "genes out of", len(gene_ids))
if verbose:
print()
print(gene_id, gene2stable(cursor, gene_id), get_description (cursor, gene_id))
# get the paralogues - only the representative for the family will have this
paralogues = get_paras (cursor, gene_id)
if not paralogues:
if verbose: print("\t not a template or no paralogues")
continue
if verbose: print("paralogues: ", paralogues)
# get _all_ exons
template_exons = gene2exon_list(cursor, gene_id)
if (not template_exons):
if verbose: print('no exons for ', gene_id)
continue
# find all template exons we are tracking in the database
for template_exon in template_exons:
if verbose: print(template_exon.exon_id)
maps = get_maps(cursor, ensembl_db_name, template_exon.exon_id,
template_exon.is_known, species=species, table='para_exon_map')
if not maps:
no_maps += 1
continue
# output to fasta:
seqname = "{0}:{1}:{2}".format('template', template_exon.exon_id, template_exon.is_known)
exon_seqs_info = get_exon_seqs (cursor, template_exon.exon_id, template_exon.is_known)
if not exon_seqs_info: continue
[exon_seq_id, pepseq, pepseq_transl_start, pepseq_transl_end,
left_flank, right_flank, dna_seq] = exon_seqs_info
if (not pepseq):
if ( template_exon.is_coding and template_exon.covering_exon <0): # this should be a master exon
print("no pep seq for", template_exon.exon_id, "coding ", template_exon.is_coding, end=' ')
print("canonical: ", template_exon.is_canonical)
print("length of dna ", len(dna_seq))
no_pepseq += 1
continue
tot += 1
sequences = {seqname:pepseq}
headers = [seqname]
for map in maps:
exon = map2exon(cursor, ensembl_db_name, map, paralogue=True)
pepseq = get_exon_pepseq (cursor,exon)
if (not pepseq):
continue
seqname = "{0}:{1}:{2}".format('para', map.exon_id_2, map.exon_known_2)
headers.append(seqname)
sequences[seqname] = pepseq
fasta_fnm = "{0}/{1}_{2}_{3}.fa".format( cfg.dir_path['scratch'], species, template_exon.exon_id, template_exon.is_known)
output_fasta (fasta_fnm, headers, sequences)
if (len(headers) <=1 ):
print("single species in the alignment (?)")
no_paralogues += 1
continue
# align
afa_fnm = "{0}/{1}_{2}_{3}.afa".format( cfg.dir_path['scratch'], species, template_exon.exon_id, template_exon.is_known)
mafftcmd = acg.generate_mafft_command (fasta_fnm, afa_fnm)
ret = subprocess.getoutput(mafftcmd)
# read in the alignment
inf = erropen(afa_fnm, "r")
if not inf:
print(gene_id)
continue
template_seq_seen = False
for record in SeqIO.parse(inf, "fasta"):
### store the alignment as bitstring
# Generate the bitmap
bs = Bits(bin='0b' + re.sub("[^0]","1", str(record.seq).replace('-','0')))
msa_bitmap = bs.tobytes()
# Retrieve information on the cognate
label, cognate_exon_id, cognate_exon_known = record.id.split(':')
if (label == 'template'):
template_seq_seen = True
# Write the bitmap to the database
#print "updating: ", template_exon.exon_id
store_or_update(cursor, "para_exon_map", {"cognate_exon_id" :cognate_exon_id,
"cognate_exon_known" :cognate_exon_known,
"exon_id" :template_exon.exon_id,
"exon_known" :template_exon.is_known},
{"msa_bitstring":MySQLdb.escape_string(msa_bitmap)})
inf.close()
ok += 1
subprocess.getoutput("rm "+afa_fnm+" "+fasta_fnm)
if verbose: print(" time: %8.3f\n" % (time()-start));
outstr = species + " done \n"
outstr += "tot: %d ok: %d \n" % (tot, ok)
outstr += "no maps %d \n" % no_pepseq
outstr += "no pepseq %d \n" % no_pepseq
outstr += "no paralogues %d \n" % no_paralogues
outstr += "\n"
print(outstr)
def gprMax_to_dzt(filename, rx, rxcomponent, centerFreq, distTx_Rx,
trace_step):
import h5py as h5
import os
import sys
import struct
import bitstruct
import datetime
from bitstring import Bits
from scipy import signal
# ------------------------------- Information specified by the user ---------------------------------------
# Specify gprMax file path name
file_path_name = filename
# Specify center frequency (MHz)
center_freq = centerFreq
# Specify Tx-Rx distance
distance = distTx_Rx
# Trace step
trace_step = trace_step
# Choose E-field component
comp = rxcomponent
# ---------------------------------------------------------------------------------------------------------
# Read gprMax data
bscan, _, _ = gprMax_Bscan(filename + '.out', rx, rxcomponent)
data = np.array(bscan)
# Read time step
#file = h5.File(filename[0:-4]+'1.out', 'r')
file = h5.File(filename + '1.out', 'r')
time_step = file.attrs['dt']
file.close()
data = (data * 32767) / np.max(np.abs(data))
data[data > 32767] = 32767
data[data < -32768] = -32768
data = np.round(data)
# Number of samples and traces
[noSamples, noTraces] = np.shape(data)
# Convert time step to ns
time_step = time_step * 10**9
# Sampling frequency (MHz)
sampling_freq = (1 / time_step) * 10**3
# Time window (ns)
time_window = time_step * noSamples
# DZT file name
fileName = filename
# Resample data to 1024 samples
data = signal.resample(data, 1024)
time_step = time_window / np.shape(data)[0]
sampling_freq = (1 / time_step) * 10**3
# ------------------------------------------------ DZT file header -----------------------------------------------------
tag = 255 # 0x00ff if header, 0xfnff for old file Header
dataOffset = 1024 # Constant 1024
noSamples = np.shape(data)[0] # Number of samples
bits = 16 # Bits per data word (8 or 16)
binaryOffset = 32768 # Binary offset (8 bit -> 128, 16 bit -> 32768)
sps = 0 # Scans per second
spm = 1 / trace_step # Scans per metre
mpm = 0 # Meters per mark
position = 0 # Position (ns)
time_window = time_window # Time window (ns)
noScans = 0 # Number of passes for 2D files
dateTime = datetime.datetime.now() # Current datetime
# Date and time created
createdSec = dateTime.second
if createdSec > 29: createdSec = 29
createdMin = dateTime.minute
createdHour = dateTime.hour
createdDay = dateTime.day
createdMonth = dateTime.month
createdYear = dateTime.year - 1980
# Date and time modified
modifiedSec = dateTime.second
if modifiedSec > 29: modifiedSec = 29
modifiedMin = dateTime.minute
modifiedHour = dateTime.hour
modifiedDay = dateTime.day
modifiedMonth = dateTime.month
modifiedYear = dateTime.year - 1980
offsetRG = 0 # Offset to range gain function
sizeRG = 0 # Size of range gain function
offsetText = 0 # Offset to text
sizeText = 0 # Size of text
offsetPH = 0 # Offset to processing history
sizePH = 0 # Size of processing history
noChannels = 1 # Number of channels
epsr = 5 # Average dielectric constant
topPosition = 0 # Top position (m)
vel = (299792458 / np.sqrt(epsr)) * 10**-9
range0 = vel * (time_window / 2) # Range (meters)
xStart = 0 # X start coordinate
xFinish = noTraces * trace_step - trace_step # X finish coordinate
servoLevel = 0 # Gain servo level
reserved = 0 # Reserved
antConfig = 0 # Antenna Configuration
setupConfig = 0 # Setup Configuration
spp = 0 # Scans per pass
noLine = 0 # Line number
yStart = 0 # Y start coordinate
yFinish = 0 # Y finish coordinate
lineOrder = 0
dataType = 2 # Data type
antennaName = 'antName'
if len(antennaName) > 14:
antennaName = antennaName[0:14]
elif len(antennaName) < 14:
antennaName = antennaName.ljust(14)
channelMask = 0 # Channel mask
fName = fileName # File name
if len(fName) > 12:
fName = fName[0:12]
elif len(fName) < 12:
fName = fName.ljust(12)
checkSum = 0 # Check sum for header
# -------------------------------------------------------------------------------------------------------------------
# ----------------------------------------- Convert to bytes and write to file --------------------------------------
# Open file to write
with open(fileName + '.dzt', 'wb') as fid:
# Write header
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, tag)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, dataOffset)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, noSamples)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, bits)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('s16<', dataStruct, 0, binaryOffset)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, sps)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, spm)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, mpm)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, position)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, time_window)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, noScans)
fid.write(dataStruct)
sec = Bits(uint=createdSec, length=5)
min = Bits(uint=createdMin, length=6)
hour = Bits(uint=createdHour, length=5)
day = Bits(uint=createdDay, length=5)
month = Bits(uint=createdMonth, length=4)
year = Bits(uint=createdYear, length=7)
b = Bits().join([year, month, day, hour, min, sec])
createDate = b.tobytes()
fid.write(bitstruct.pack('>r32<', createDate))
sec = Bits(uint=modifiedSec, length=5)
min = Bits(uint=modifiedMin, length=6)
hour = Bits(uint=modifiedHour, length=5)
day = Bits(uint=modifiedDay, length=5)
month = Bits(uint=modifiedMonth, length=4)
year = Bits(uint=modifiedYear, length=7)
b = Bits().join([year, month, day, hour, min, sec])
modifiedDate = b.tobytes()
fid.write(bitstruct.pack('>r32<', modifiedDate))
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, offsetRG)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, sizeRG)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, offsetText)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, sizeText)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, offsetPH)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, sizePH)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, noChannels)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, epsr)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, topPosition)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, range0)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, xStart)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, xFinish)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, servoLevel)
fid.write(dataStruct)
dataStruct = bytearray(3)
bitstruct.pack_into('r24<', dataStruct, 0, reserved)
fid.write(dataStruct)
dataStruct = bytearray(1)
bitstruct.pack_into('u8<', dataStruct, 0, antConfig)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, setupConfig)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, spp)
fid.write(dataStruct)
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, noLine)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, yStart)
fid.write(dataStruct)
dataStruct = bytearray(4)
bitstruct.pack_into('f32<', dataStruct, 0, yFinish)
fid.write(dataStruct)
dataStruct = bytearray(1)
bitstruct.pack_into('u8<', dataStruct, 0, lineOrder)
fid.write(dataStruct)
dataStruct = bytearray(1)
bitstruct.pack_into('r8<', dataStruct, 0, dataType)
fid.write(dataStruct)
fid.write(bitstruct.pack('t14<', antennaName))
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, channelMask)
fid.write(dataStruct)
fid.write(bitstruct.pack('t12<', fName))
dataStruct = bytearray(2)
bitstruct.pack_into('u16<', dataStruct, 0, checkSum)
fid.write(dataStruct)
# Move to 1024 to write data
fid.seek(dataOffset)
data = data + binaryOffset
data = np.array(data, dtype='<H')
fid.write(data.T.astype('<H').tobytes())
# Close file
fid.close()
print('Dzt file has been written!')
