def filterData(reconData, logDir, fileEnding, samplingInterval, coefficients):
# Filter the reconstructed data
#print "Filtering the data..."
filteredData = []
time, x, y, z = splitData(reconData)
xFiltered = scipy.signal.convolve(coefficients, x)
yFiltered = scipy.signal.convolve(coefficients, y)
zFiltered = scipy.signal.convolve(coefficients, z)
numberToDrop = len(coefficients) - 1
time = time[numberToDrop:]
xFiltered = xFiltered[numberToDrop:]
yFiltered = yFiltered[numberToDrop:]
zFiltered = zFiltered[numberToDrop:]
for i, v in enumerate(time):
sample = Sample()
sample.time = time[i]
sample.position.x = xFiltered[i]
sample.position.y = yFiltered[i]
sample.position.z = zFiltered[i]
filteredData.append(sample)
filteredData = filteredData[:-200]
exportData(logDir + "FilteredData_" + fileEnding + ".txt", filteredData)
return [filteredData]
def __init__(self,
Actor,
Critic,
env,
lr=0.001,
n_actions=1,
model_file=None):
self.env = env
self.lr = lr
self.actor = Actor
self.critic = Critic
self.critic_train = self.critic(trainable=True)
self.action_bound = [-env.action_space.high, env.action_space.high]
self.n_features = env.observation_space.shape[0]
self.n_actions = n_actions
self.actor_train = self.actor(env,
n_actions,
self.action_bound,
trainable=True)
self.actor_old = self.actor(env,
n_actions,
self.action_bound,
trainable=False)
self.actor_net = self.actor_train.build_net(
input_shape=self.n_features)
self.critic_net = self.critic_train.build_net(
input_shape=self.n_features)
self.actor_oldnet = self.actor_old.build_net(
input_shape=self.n_features)
# self.pi = None
# self.pi_old = None
self.Sample = Sample(env, self.actor_train, self.critic_train)
def create_Samples_from_header(line):
"""Creates sample objects for input file.
It uses CN and CP input lists from users.
It creates a sample object for each one containing the name
and the column index for genotype, depth amd var/depth (ratio) columns
"""
samples_name_list = []
sample_return_list = []
geno_sufix = '.replicate1_Genotype'
depth_sufix = '.replicate1_Depth'
ratio_sufix = '.replicate1_Var/Depth'
columns = line.split('|')
for i in columns:
res = re.search('(.*)\.replicate1_Genotype$', i)
if res: samples_name_list.append(res.group(1))
for sample_name in samples_name_list:
genotype_col = columns.index(sample_name + geno_sufix)
depth_col = columns.index(sample_name + depth_sufix)
ratio_col = columns.index(sample_name + ratio_sufix)
new_sample = Sample(sample_name, genotype_col, depth_col, ratio_col)
#We create and set a VariantCounter object to the Sample.
new_sample.set_variant_counter(VariantCounter())
sample_return_list.append(new_sample)
return sample_return_list
def filterData(reconData, logDir, fileEnding, samplingInterval, coefficients): # Filter the reconstructed data #print "Filtering the data..." filteredData = [] time, x, y, z = splitData(reconData) xFiltered = scipy.signal.convolve(coefficients, x) yFiltered = scipy.signal.convolve(coefficients, y) zFiltered = scipy.signal.convolve(coefficients, z) numberToDrop = len(coefficients) - 1 time = time[numberToDrop:] xFiltered = xFiltered[numberToDrop:] yFiltered = yFiltered[numberToDrop:] zFiltered = zFiltered[numberToDrop:] for i, v in enumerate(time): sample = Sample() sample.time = time[i] sample.position.x = xFiltered[i] sample.position.y = yFiltered[i] sample.position.z = zFiltered[i] filteredData.append(sample) filteredData = filteredData[:-200] exportData(logDir + "FilteredData_" + fileEnding + ".txt", filteredData) return [filteredData]
def __call__(self):
""" Main function of the script """
start_time = time()
print ("Start parsing files: {} chunks to be parsed".format(len(self.seq_R1)))
# For double indexing
if self.idx2:
self.double_index_parser()
# For simple indexing
else:
self.simple_index_parser()
# Flush remaining content in sample buffers
Sample.FLUSH_ALL()
# Write a report
print ("Generate_a csv report")
with open ("Quade_report.csv", "wb") as report:
report.write ("Program {}\tDate {}\n\n".format(self.VERSION,str(datetime.today())))
for descr, value in Sample.REPORT():
report.write ("{}\t{}\n".format(descr, value))
print ("Done in {}s".format(round(time()-start_time, 3)))
return(0)
def executeAlgorithm(self):
self.reconstructedSignal = []
nextTimeToRecord = 0
timeDiff = self.rawSignal[0].sample.time % self.samplingInterval
if timeDiff == 0:
self.reconstructedSignal.append( deepcopy(self.rawSignal[0].sample) )
nextTimeToRecord = self.rawSignal[0].sample.time + \
self.samplingInterval
else:
change = self.samplingInterval - timeDiff
newSample = Sample()
newSample.time = self.rawSignal[0].sample.time + change
newSample.position = deepcopy(self.rawSignal[0].sample.position)
self.reconstructedSignal.append(newSample)
nextTimeToRecord = newSample.time + self.samplingInterval
for index, predictionSample in enumerate(self.rawSignal):
if predictionSample.sample.time == nextTimeToRecord:
self.reconstructedSignal.append( deepcopy(predictionSample.sample) )
nextTimeToRecord = nextTimeToRecord + self.samplingInterval
elif predictionSample.sample.time > nextTimeToRecord:
while predictionSample.sample.time > nextTimeToRecord:
deltaTime = nextTimeToRecord - self.rawSignal[index-1].sample.time
deltaTimeVector = Vector(deltaTime, deltaTime, deltaTime)
deltaPosition = self.rawSignal[index-1].velocity * deltaTimeVector
newPosition = self.rawSignal[index-1].sample.position + deltaPosition
newSample = Sample(nextTimeToRecord, newPosition)
self.reconstructedSignal.append( newSample )
nextTimeToRecord = nextTimeToRecord + self.samplingInterval
if predictionSample.sample.time == nextTimeToRecord:
self.reconstructedSignal.append( deepcopy(predictionSample.sample) )
nextTimeToRecord = nextTimeToRecord + self.samplingInterval
def create_Samples_from_header(line):
"""Creates sample objects for input file.
It uses CN and CP input lists from users.
It creates a sample object for each one containing the name
and the column index for genotype, depth amd var/depth (ratio) columns
"""
samples_name_list = []
sample_return_list = []
geno_sufix = '.replicate1_Genotype'
depth_sufix = '.replicate1_Depth'
ratio_sufix = '.replicate1_Var/Depth'
columns = line.split('|')
for i in columns:
res = re.search('(.*)\.replicate1_Genotype$', i)
if res: samples_name_list.append(res.group(1))
for sample_name in samples_name_list:
genotype_col = columns.index(sample_name+geno_sufix)
depth_col = columns.index(sample_name+depth_sufix)
ratio_col = columns.index(sample_name+ratio_sufix)
new_sample = Sample(sample_name, genotype_col, depth_col, ratio_col)
#We create and set a VariantCounter object to the Sample.
new_sample.set_variant_counter(VariantCounter())
sample_return_list.append(new_sample)
return sample_return_list
def calculateEstSample(self, interpolationSample, currentTime):
estimatedSample = Sample()
estimatedSample.time = currentTime
estimatedSample.position = self.calculateEstPosition(interpolationSample, currentTime)
return estimatedSample
def __init__(self, mx):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = pow(10, 32)
self.hi = pow(-10, 32)
self._some = Sample(mx)
self.w = 1
def __init__(self, m=0):
self.n = 0
self.mu = 0
self.m2 = 0
self.sd = 0
self.lo = math.pow(10, 32)
self.hi = -1 * math.pow(10, 32)
self._some = Sample(m)
self.w = 1
def addSample(self, file):
try:
s = Sample(file)
s.getParameters()
return s
except Exception as e:
print(f"Import Error. {file} not added to project" , e)
return None
def loadSamples(self, lineSplit):
i = 0
for item in lineSplit:
if i == 0:
S = Sample(item)
self.Samples.append(S)
#print "Creating " + S.toStr()
else:
S.addLipid(self.lipidIndexMap[i - 1], float(item))
i = i + 1
def findFirstSample(self):
timeDiff = self.rawSignal[0].sample.time % self.samplingInterval
if timeDiff == 0:
return deepcopy(self.rawSignal[0].sample)
else:
change = self.samplingInterval - timeDiff
newSample = Sample()
newSample.time = self.rawSignal[0].sample.time + change
newSample.position = deepcopy(self.rawSignal[0].sample.position)
return newSample
def findFirstSample(self):
timeDiff = self.rawSignal[0].sample.time % self.samplingInterval
if timeDiff == 0:
return deepcopy(self.rawSignal[0].sample)
else:
change = self.samplingInterval - timeDiff
newSample = Sample()
newSample.time = self.rawSignal[0].sample.time + change
newSample.position = deepcopy(self.rawSignal[0].sample.position)
return newSample
def shiftData(self):
tempData = []
firstTime = self.rawData[0].time
for sample in self.rawData:
if sample.time >= firstTime:
newSample = Sample()
newSample.time = sample.time - firstTime + ( 10 * self.numberOfPaddingSamples)
newSample.position = copy( sample.position )
tempData.append(newSample)
self.rawData = tempData
def load(self, path):
samples_path = os.path.join(path, 'samples')
if not os.path.exists(samples_path):
os.mkdir(samples_path)
items = os.listdir(samples_path)
samples = [item for item in items if item.endswith('.txt')]
for s in samples:
sample = Sample()
sample.load(os.path.join(samples_path, s))
self.add(sample)
def shiftData(self):
tempData = []
firstTime = self.rawData[0].time
for sample in self.rawData:
if sample.time >= firstTime:
newSample = Sample()
newSample.time = sample.time - firstTime + (
10 * self.numberOfPaddingSamples)
newSample.position = copy(sample.position)
tempData.append(newSample)
self.rawData = tempData
def scaleData(self):
tempData = []
for sample in self.rawData:
newSample = Sample()
newSample.time = sample.time
newSample.position = (sample.position + self.scalingFactorA)
newSample.position.x = newSample.position.x * self.scalingFactorB.x
newSample.position.y = newSample.position.y * self.scalingFactorB.y
newSample.position.z = newSample.position.z * self.scalingFactorB.z
newSample.position = newSample.position - Vector(1, 1, 1)
tempData.append(newSample)
self.rawData = tempData
def scaleData(self):
tempData = []
for sample in self.rawData:
newSample = Sample()
newSample.time = sample.time
newSample.position = ( sample.position + self.scalingFactorA )
newSample.position.x = newSample.position.x * self.scalingFactorB.x
newSample.position.y = newSample.position.y * self.scalingFactorB.y
newSample.position.z = newSample.position.z * self.scalingFactorB.z
newSample.position = newSample.position - Vector(1,1,1)
tempData.append(newSample)
self.rawData = tempData
def recordSample(self, env):
s = Sample()
s.time = env.time
s.simcarPos = env.simcar.p.myclone()
s.simcarHeading = env.simcar.h.myclone()
s.simcarFracIndex = env.simcar.fracIndex
s.simcarSpeed = env.simcar.speed
s.simcarRoadIndex = env.simcar.roadIndex
def getMedleyDB(xml_path):
tree = etree.parse(xml_path)
root = tree.getroot()
db_path = root.find("./databaseFolderPath").text
mixes, accs, vocals = list(), list(), list()
tracks = root.xpath(".//track")
for track in tracks:
instrument_paths = list()
# Mix together vocals, if they exist
vocal_tracks = track.xpath(".//instrument[instrumentName='Voice']/relativeFilepath") + \
track.xpath(".//instrument[instrumentName='Voice']/relativeFilepath") + \
track.xpath(".//instrument[instrumentName='Voice']/relativeFilepath")
if len(
vocal_tracks
) > 0: # If there are vocals, get their file paths and mix them together
vocal_track = Input.Input.add_audio(
[db_path + os.path.sep + f.text for f in vocal_tracks],
"vocalmix")
instrument_paths.append(vocal_track)
vocals.append(Sample.from_path(vocal_track))
else: # Otherwise append duration of track so silent input can be generated later on-the-fly
duration = float(
track.xpath("./instrumentList/instrument/length")[0].text)
vocals.append(duration)
# Mix together accompaniment, if it exists
acc_tracks = track.xpath(
".//instrument[not(instrumentName='Voice') and not(instrumentName='Mix') and not(instrumentName='Instrumental')]/relativeFilepath"
) #TODO # We assume that there is no distinction between male/female here
if len(
acc_tracks
) > 0: # If there are vocals, get their file paths and mix them together
acc_track = Input.Input.add_audio(
[db_path + os.path.sep + f.text for f in acc_tracks], "accmix")
instrument_paths.append(acc_track)
accs.append(Sample.from_path(acc_track))
else: # Otherwise append duration of track so silent input can be generated later on-the-fly
duration = float(
track.xpath("./instrumentList/instrument/length")[0].text)
accs.append(duration)
# Mix together vocals and accompaniment
mix_track = Input.Input.add_audio(instrument_paths, "totalmix")
mixes.append(Sample.from_path(mix_track))
return [mixes, accs, vocals]
def simple_index_parser (self):
# Iterate over fastq chunks for sequence and index reads
for n, (R1, R2, I1) in enumerate (zip(self.seq_R1, self.seq_R2, self.index_R1)):
print("Start parsing chunk {}/{}".format(n+1, len(self.seq_R1)))
# Init FastqReader generators
R1_gen = FastqReader(R1)
R2_gen = FastqReader(R2)
I1_gen = FastqReader(I1)
# Iterate over reads in fastq files until exhaustion
try:
while True:
read1 = R1_gen.next()
read2 = R2_gen.next()
index1 = I1_gen.next()
# Extract index and molecular sequences from index reads
index = index1[self.idx1_pos["start"]:self.idx1_pos["end"]]
molecular = index1.seq[self.mol1_pos["start"]:self.mol1_pos["end"]]
# Identify sample and verify index quality
Sample.FINDER (read1,read2, index, molecular)
except StopIteration as E:
print(E)
print("\tEnd of chunk {}".format(n+1))
def Open(self):
files = []
names = map(
lambda ms: '_'.join(
filter(None, (self.name, Mode.getName(ms[0]),
Sample.getName(ms[1])))), self.modesample)
for f in os.listdir('output'):
if not f.endswith('.root'):
continue
if filter(lambda n: f.startswith(n), names):
files.append('output/' + f)
if len(files) > 1:
self._close = 'delete _chain;'
return '\n'.join(['', '_chain = new TChain("result");'] +
['_chain->Add("{0}");'.format(f)
for f in files] + ['return _chain;', ''])
elif len(files) == 1:
self._close = '_file->Close();'
return '\n'.join([
'', '_file = TFile::Open("{0}");'.format(files[0]),
'TTree* tree = (TTree*)_file->Get("result");', 'return tree;'
''
])
else:
raise IndexError('Couldn\'t find files for ' + self.title)
def parse_binary(path) -> list:
name = path.split('/')[-1][0:4]
collection = []
with open(path, 'rb') as file:
n = file.read(8)
n = struct.unpack('<q', n)[0]
while True:
time = file.read(8)
time = struct.unpack('<q', time)[0]
pos = np.zeros([3])
mat = np.zeros([3, 3])
for i in range(0, 3):
tmp = file.read(8)
tmp = struct.unpack('<d', tmp)[0]
pos[i] = tmp
for i in range(0, 3):
for j in range(0, 3):
tmp = file.read(8)
tmp = struct.unpack('<d', tmp)[0]
mat[i, j] = tmp
pos_f = file.tell()
collection.append(Sample(name, time, pos, mat))
if not file.read(1):
return collection
else:
file.seek(pos_f)
def setup_samples(parent_dir):
"""
Reads samples directories and creates objects for each sample.
"""
print "Reading directory %s ..." % parent_dir
# Container to keep sample objects
samples = []
# Get subdirectories in parent dir
subdirs = [
os.path.join(parent_dir, s) for s in os.listdir(parent_dir)
if os.path.isdir(os.path.join(parent_dir, s))
]
for sd in subdirs:
# Loop files in sample directory
abs_sample_path = os.path.abspath(os.path.join(parent_dir, sd))
# Create sample object
sample = Sample(abs_sample_path, os.path.abspath(parent_dir))
# Add to samples collection
samples.append(sample)
# Return all samples
return samples
def getVocalFMA(database_path, audio_path=None):
if audio_path is None:
audio_path = database_path
track_csv = os.path.join(database_path, "fma_metadata", "raw_tracks.csv")
sample_list = list()
with open(track_csv, 'rb') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
if not int(row["track_instrumental"]):
track_id = int(row["track_id"])
filename = '{:06d}'.format(track_id) + ".mp3"
folder_id = track_id // 1000
foldername = '{:03d}'.format(folder_id)
audio_file = os.path.join(audio_path, "fma_full", foldername, filename)
print("Reading in metadata of file " + audio_file)
try:
sample = Sample.from_path(audio_file)
except Exception as e:
print("Skipping sample at path " + audio_file)
print(e)
continue
sample_list.append(sample)
return sample_list
def home():
context = defaultdict(lambda: "")
if request.method == "POST":
sample = Sample(request.form["text"],
prediction_score=predict(request.form["text"]))
context["sample"] = sample
return render_template("home.html", context=context)
def from_folder(cls, folder_path, tag):
slist = cls()
files = glob.glob(folder_path + "*.*")
for file in files:
sample = Sample.from_file(file, tag)
slist.append(sample)
return slist
def test_isOverLimitTime(self):
# 境界確認
# datetimeのMockがまともに動かない。原始的に行く
start = dt.datetime.now()
actual = False
proc = dt.datetime.now()
limit = 2
while (not actual):
actual = Sample.isOverLimitTime(self, start, limit)
proc = dt.datetime.now()
# 処理時間分で微妙にずれるけど、そこまで厳密に見る気もないのでいっか。。。
self.assertGreater((proc - start).total_seconds(), limit)
#with patch('Sample.datetime.datetime') as mock:
# from datetime import datetime
#nowを10秒経過後にしておく
# mock.now.return_valie = start + testdatetime.timedelta(seconds=10)
#end - start = limit
#まともにMockできそうにないからほどほどにしておく
#end - start > limit
#actual = Sample.isOverLimitTime(self, start, 9)
#self.assertEqual(True, actual)
#end - start < limit
#actual = Sample.isOverLimitTime(self, start, 11)
#self.assertEqual(False, actual)
pass
def main(eval_name, ml):
uc = UserController()
followers = uc.getFollowers(eval_name)
directory = "followers_{}".format(eval_name)
full_directory = "data/{}".format(directory)
if not os.path.exists(full_directory):
os.makedirs(full_directory)
for name in followers:
user = User(name, "{}/unknown".format(directory, name))
if user.valid:
user.writeFile()
path = "data/followers_{}".format(eval_name)
follower_files = [f for f in listdir(path) if isfile(join(path, f))]
result = {}
for i in range(2):
for j in range(2):
for k in range(2):
for l in range(2):
type = [i, j, k, l]
result[decodeType(type)] = []
for file in follower_files:
follower = Sample("{}/{}".format(path, file))
type = ml.predict(follower.data)
result[decodeType(type)].append(follower.twitName)
with open("evaluated/follower_types_{}.json".format(eval_name), "w") as f:
json.dump(result, f)
def get_data(self):
print('Beggining audio file reading...')
for x in range(0 , len(self.genres)):
genre_path = self.main_path + '\\' + self.genres[x]
drop_indices = sorted(random.sample(range(1,100),20))
self.excluded.append(drop_indices)
sample_list = list(range(0,100))
for i in reversed(drop_indices):
del sample_list[i]
for i in range(0,self.n_samples):
current_count = '.' + '%05d' % i
song_path = genre_path + '\\' + self.genres[x] + current_count + '.au'
sample = Sample(song_path)
self.audio_data.extend(sample.final_data)
for y in range(0,sample.image_number):
self.labels.append(x)
if i%10 == 0:
print('Percentage Complete :' ,
int(((x*100)+i)/(len(self.genres)*100)*100) , '%')
print('Audio read succesfully')
print('Saving droped indices')
self.excluded = np.array(self.excluded)
print(self.excluded)
try:
np.save('C:\\Users\\nick\\Desktop\\Final_Project\\excluded_indices',
self.excluded)
except Exception as e:
print('Write Failed')
self.audio_data = np.array(self.audio_data)
self.labels = np.array(self.labels)
self.labels_onehot = (np.arange(len(self.genres)) ==
self.labels[: , None]).astype(int)
def create_sample(db_path, instrument_node):
path = db_path + os.path.sep + instrument_node.xpath(
"./relativeFilepath")[0].text
sample_rate = int(instrument_node.xpath("./sampleRate")[0].text)
channels = int(instrument_node.xpath("./numChannels")[0].text)
duration = float(instrument_node.xpath("./length")[0].text)
return Sample(path, sample_rate, channels, duration)
def new(self, params):
os.makedirs(params.out_dir, exist_ok=True)
self.params = params
w.write(params.out_dir + "/params.json",
w.pretty_json(params.to_json()))
self.sample = Sample().new(params)
sample_file = params.out_dir + "/sample.fasta"
w.write(sample_file, w.fasta(self.sample))
art_prefix = params.out_dir + "/art"
art = os.environ['ART_ILLUMINA']
subprocess.run([
art, "--in", sample_file, "--out", art_prefix, "--rndSeed",
str(params.seed)
] + params.art_flags,
stdout=subprocess.DEVNULL)
self.art_output = r.read(art_prefix + ".aln", r.aln(params.take_ref))
self.instance = Instance().new(params, self.art_output)
w.write(params.out_dir + "/instance.json",
w.json(self.instance.to_json()))
w.write(params.out_dir + "/instance.txt",
w.text(self.instance.to_text()))
w.write(params.out_dir + "/instance.stats.json",
w.json(self.instance.stats()))
return self
def create_Samples_from_header(line, sample_list):
"""Creates sample objects for input file.
It uses CN and CP input lists from users.
It creates a sample object for each one containing the name
and the column index for genotype, depth amd var/depth (ratio) columns
"""
sample_object_list = []
geno_sufix = '.replicate1_Genotype'
depth_sufix = '.replicate1_Depth'
ratio_sufix = '.replicate1_Var/Depth'
columns = line.split('|')
#if not set(sample_list).issubset(set(columns)): quit("ERROR: Specified samples: %s couldn't be found in input annotation file." %sample_list)
for sample_name in sample_list:
try:
genotype_col = columns.index(sample_name + geno_sufix)
depth_col = columns.index(sample_name + depth_sufix)
ratio_col = columns.index(sample_name + ratio_sufix)
new_sample = Sample(sample_name, genotype_col, depth_col,
ratio_col)
sample_object_list.append(new_sample)
except ValueError:
print "ERROR: Specified samples: %s couldn't be found in input annotation file." % sample_list
quit()
return sample_object_list
def getDSDFilelist(xml_path):
tree = etree.parse(xml_path)
root = tree.getroot()
db_path = root.find("./databaseFolderPath").text
tracks = root.findall(".//track")
train_vocals, test_vocals, train_mixes, test_mixes, train_accs, test_accs = list(), list(), list(), list(), list(), list()
for track in tracks:
# Get mix and vocal instruments
vocals = create_sample(db_path, track.xpath(".//instrument[instrumentName='Voice']")[0])
mix = create_sample(db_path, track.xpath(".//instrument[instrumentName='Mix']")[0])
[acc_path] = subtract_audio([mix.path], [vocals.path])
acc = Sample(acc_path, vocals.sample_rate, vocals.channels, vocals.duration) # Accompaniment has same signal properties as vocals and mix
if track.xpath("./databaseSplit")[0].text == "Training":
train_vocals.append(vocals)
train_mixes.append(mix)
train_accs.append(acc)
else:
test_vocals.append(vocals)
test_mixes.append(mix)
test_accs.append(acc)
return [train_mixes, train_accs, train_vocals], [test_mixes, test_accs, test_vocals]
def go_fetch(kfold_n: int = 5):
# NOTE: File structure relative to this script is expected to be as follows.
# .
# +-- Captures
# | +-- Make x_0
# | | +-- Model y_0
# | | | +-- ModelYear z_0
# | | | | +-- Samples
# | | | | | +-- sample = re.match('loggerProgram[\d]+.log', a_file_in_this_folder)
# | +-- Make x_1.... etc.
# +-- Some folder
# | +-- The directory with these scripts
# | | +-- this_script.py
script_dir: str = getcwd()
chdir("../../")
if not path.exists("Captures"):
# Make sure your local directory structure matches the example above. If not... adjust accordingly
print("Error finding Captures folder. Please check the relative path between this script and Captures.")
print("See the source of go_fetch() in FileBoi.py for an example of the expected relative paths.")
chdir(script_dir)
quit()
chdir("Captures")
root_dir = getcwd()
make: str = ""
model: str = ""
year: str = ""
current_vehicle = []
sample_dict = {}
for dirName, subdirList, fileList in walk(root_dir, topdown=True):
this_dir = path.basename(dirName)
if len(subdirList) == 0:
if len(current_vehicle) == 3:
make = current_vehicle[0]
model = current_vehicle[1]
year = current_vehicle[2]
elif len(current_vehicle) == 2:
model = current_vehicle[0]
year = current_vehicle[1]
elif len(current_vehicle) == 1 and current_vehicle != "":
year = current_vehicle[0]
for file in fileList:
# Check if this file name matches the expected name for a CAN data sample. If so, create new Sample
m = re.match('loggerProgram[\d]+.log', file)
if m:
if not (make, model, year) in sample_dict:
sample_dict[(make, model, year)] = []
this_sample_index = str(len(sample_dict[(make, model, year)]))
this_sample = Sample(make=make, model=model, year=year, sample_index=this_sample_index,
sample_path=dirName + "/" + m.group(0), kfold_n=kfold_n)
sample_dict[(make, model, year)].append(this_sample)
current_vehicle = []
else:
if this_dir == "Captures":
continue
current_vehicle.append(this_dir)
chdir(script_dir)
return sample_dict
def importData(self, fileName):
input = open( fileName, 'r' )
for line in input:
tokens = line.split('\t')
sample = Sample()
minutes = int( tokens[self.minutePosition] )
seconds = int( tokens[self.secondPosition] )
milliseconds = int( tokens[self.millisecondPosition] )
sample.time = minutes * 60 * 1000 + seconds * 1000 + milliseconds
sample.position.x = float( tokens[self.xPosition] )
sample.position.y = float( tokens[self.yPosition] )
sample.position.z = float( tokens[self.zPosition] )
self.rawData.append(sample)
input.close()
def from_folder(cls, folder_path, tag):
slist = cls()
files = glob.glob(folder_path + "*.*")
for file in files:
sample = Sample.from_file(file, tag)
slist.append(sample)
return slist
def importData(self, fileName):
input = open(fileName, 'r')
for line in input:
tokens = line.split('\t')
sample = Sample()
minutes = int(tokens[self.minutePosition])
seconds = int(tokens[self.secondPosition])
milliseconds = int(tokens[self.millisecondPosition])
sample.time = minutes * 60 * 1000 + seconds * 1000 + milliseconds
sample.position.x = float(tokens[self.xPosition])
sample.position.y = float(tokens[self.yPosition])
sample.position.z = float(tokens[self.zPosition])
self.rawData.append(sample)
input.close()
def executeAlgorithm(self):
self.reconstructedSignal = []
nextTimeToRecord = 0
timeDiff = self.rawSignal[0].sample.time % self.samplingInterval
if timeDiff == 0:
self.reconstructedSignal.append( deepcopy(self.rawSignal[0].sample) )
nextTimeToRecord = self.rawSignal[0].sample.time + \
self.samplingInterval
else:
change = self.samplingInterval - timeDiff
newSample = Sample()
newSample.time = self.rawSignal[0].sample.time + change
newSample.position = deepcopy(self.rawSignal[0].sample.position)
self.reconstructedSignal.append(newSample)
nextTimeToRecord = newSample.time + self.samplingInterval
interpolationSample = PredictionSample()
interpolationSample.sample = self.reconstructedSignal[0]
interpolationSample.velocity = self.rawSignal[0].velocity
for index, predictionSample in enumerate(self.rawSignal):
if predictionSample.sample.time == nextTimeToRecord:
estimatedSample = self.calculateEstSample(interpolationSample, \
nextTimeToRecord)
self.reconstructedSignal.append(estimatedSample)
targetSample = self.findTarget(predictionSample)
interpolationSample = self.findInterpolationSample(estimatedSample, \
targetSample)
nextTimeToRecord = nextTimeToRecord + self.samplingInterval
elif predictionSample.sample.time > nextTimeToRecord:
while predictionSample.sample.time > nextTimeToRecord:
estimatedSample = self.calculateEstSample(interpolationSample, \
nextTimeToRecord)
self.reconstructedSignal.append(estimatedSample)
nextTimeToRecord = nextTimeToRecord + self.samplingInterval
if predictionSample.sample.time == nextTimeToRecord:
estimatedSample = self.calculateEstSample(interpolationSample, \
nextTimeToRecord)
self.reconstructedSignal.append(estimatedSample)
targetSample = self.findTarget(predictionSample)
interpolationSample = self.findInterpolationSample(estimatedSample, \
targetSample)
nextTimeToRecord = nextTimeToRecord + self.samplingInterval
else:
targetSample = self.findTarget(predictionSample)
interpolationSample = self.findInterpolationSample(self.reconstructedSignal[-1], \
targetSample)
def interpolate(self, samples, currentTime):
newSample = Sample()
newSample.time = currentTime
time = [ samples[0].time, samples[1].time ]
x = [ samples[0].position.x, samples[1].position.x ]
y = [ samples[0].position.y, samples[1].position.y ]
z = [ samples[0].position.z, samples[1].position.z ]
interpolationFunction = interpolate.interp1d( time, x )
newSample.position.x = interpolationFunction(currentTime).flatten()[0]
interpolationFunction = interpolate.interp1d( time, y )
newSample.position.y = interpolationFunction(currentTime).flatten()[0]
interpolationFunction = interpolate.interp1d( time, z )
newSample.position.z = interpolationFunction(currentTime).flatten()[0]
return newSample
def interpolate(self, samples, currentTime):
newSample = Sample()
newSample.time = currentTime
time = [samples[0].time, samples[1].time]
x = [samples[0].position.x, samples[1].position.x]
y = [samples[0].position.y, samples[1].position.y]
z = [samples[0].position.z, samples[1].position.z]
interpolationFunction = interpolate.interp1d(time, x)
newSample.position.x = interpolationFunction(currentTime).flatten()[0]
interpolationFunction = interpolate.interp1d(time, y)
newSample.position.y = interpolationFunction(currentTime).flatten()[0]
interpolationFunction = interpolate.interp1d(time, z)
newSample.position.z = interpolationFunction(currentTime).flatten()[0]
return newSample
def samples(self):
"""
Return iterator of all samples collected during this visit.
"""
linkage_query = '"{}"[linkage.collected_during]'.format(self.id)
query = iHMPSession.get_session().get_osdf().oql_query
for page_no in count(1):
res = query(Visit.namespace, linkage_query, page=page_no)
res_count = res['result_count']
for doc in res['results']:
yield Sample.load_sample(doc)
res_count -= len(res['results'])
if res_count < 1:
break
def run(file_path):
dis = Dissector()
parser = Parser()
extrator = Extractor()
if os.path.isdir(file_path):
dir_files_list = os.listdir(file_path)
for files in dir_files_list:
sample = Sample(os.path.join(file_path, files))
dis.extract_file(sample)
parser.parse(sample)
extrator.extract(sample)
sample.print_info()
else:
sample = Sample(file_path)
dis.extract_file(sample)
parser.parse(sample)
extrator.extract(sample)
sample.print_info()
#!/usr/bin/env python
#atlas_setup root
from ROOT import *
from Plotter import Plotter
from Sample import Sample
from Logging import *
import array
test = True
if test:
x = Sample( FileName = 'rootFiles/mtestReadoutAnalysis_siliconDigitPlots.root', Type = 'h', Sumw2 = False, Label = 'ReadoutAna_siliconTruthSanityPlots')
print x
x.Type2D = 'COLZ'
x.LineColour = 4
samples = [x]
plots = Plotter(samples)
plots.PlotLegend = False
# PER EVENT HISTOS
fname = "plots/mtestReadoutAnalysis/siliconDigitPlots"
#!/usr/bin/env python
#atlas_setup root
from ROOT import *
from Plotter import Plotter
from Sample import Sample
from Logging import *
import array
test = True
if test:
StrawSanity = Sample( FileName = 'rootFiles/mtestGunAnalysis_strawTruthSanityPlots.root', Type = 'h', Sumw2 = False, Label = 'straws')
print StrawSanity
StrawSanity.Type2D = 'COLZ'
StrawSanity.LineColour = 4
samples = [StrawSanity]
plots = Plotter(samples)
plots.PlotLegend = False
# PER EVENT HISTOS
fname = "plots/mtestGunAnalysis/StrawSanity"
def i_validate(self):
"""
Interface to validate a classification. It going to rasterize the validation shapefile and the
classification shapefile with :func:`layer_rasterization`. Next, to compare pixel by pixel, the classification
quality to built a confusion matrix in a csv file.
"""
# Variable to convert the input classname to an individual interger
# Only for the validate sample
class_validate = 0
complete_validate_shp = os.path.dirname(self.valid_shp[0][0]) + '/validate.shp'
# TODO: Set this method in the Precision_moba class
# Processing to rasterize the validate shapefile. 1) Merge sahpefiles 2) Rasterization
for val in self.valid_shp:
if class_validate != 2:
# Grassland to 1
if (class_validate !=3 and len(self.out_fieldname_carto) != 4+2) or len(self.out_fieldname_carto) == 4+2:
# To the level 3 with woodeen to 4 and 5
#
# Self.valid_shp is a list of list. In this variable there is :
# [Shapefile path, fieldname classes, classnames]
opt = {}
opt['Remove'] = 1 # To overwrite
# Create a raster to valide the classification
# First time, create a new shapefile with a new field integer
sample_val = Sample(val[0], self.path_area, 1, **opt)
opt['add_fieldname'] = 1
opt['fieldname'] = 'CLASS_CODE'
opt['class'] = str(class_validate) # Add integer classes
# Set the new shapefile
val[0] = val[0][:-4] + '_.shp'
val[1] = opt['fieldname']
val[2] = opt['class']
# Complete the new shapefile
sample_val.fill_sample(val[0], 0, **opt)
# Second time, merge the validate shapefile
if class_validate == 0:
process_tocall_merge = ['ogr2ogr', '-overwrite', complete_validate_shp, val[0]]
elif class_validate > 0:
process_tocall_merge = ['ogr2ogr', '-update', '-append', complete_validate_shp, \
val[0], '-nln', os.path.basename(complete_validate_shp[:-4])]
subprocess.call(process_tocall_merge)
# Increrment variable
class_validate = self.valid_shp.index(val) + 1
# Compute precision of the classification
valid = Precision_moba(self.path_area, self.path_folder_dpt)
valid.complete_validation_shp = complete_validate_shp
valid.ex_raster = self.raster_path[0]
# TODO: Call the RasterSat_by_Date class here instead of the Precision_moba class
valid.preprocess_to_raster_precision(self.output_name_moba, 'FBPHY_SUB') # To the classification's data
valid.preprocess_to_raster_precision(complete_validate_shp, val[1]) # To the validation's data
# Compute precision on the output classification
valid.confus_matrix(valid.complete_img[0].raster_data(valid.img_pr[0])[0], \
valid.complete_img[1].raster_data(valid.img_pr[1])[0])
print(cn[0])
elif rfc2459.id_at_organizationName == at:
on = decode(r.getComponentByName('value'),asn1Spec=rfc2459.X520OrganizationName())
print(on[0].getComponent())
elif rfc2459.id_at_organizationalUnitName == at:
ou = decode(r.getComponentByName('value'),asn1Spec=rfc2459.X520OrganizationalUnitName())
print(ou[0].getComponent())
elif rfc2459.id_at_commonName == at:
cn = decode(r.getComponentByName('value'),asn1Spec=rfc2459.X520CommonName())
print(cn[0].getComponent())
else:
print at
if __name__=="__main__":
data=open(source_path+"/Test_files/certificate5.codex","rb").read()
sample=Sample()
sample.setBinary(data)
modules={}
pfm=PEFileModule()
modules[pfm.getName()]=pfm
plug=CertficatePlug()
plug.setModules(modules)
plug.setSample(sample)
res=plug.process()
print(res)
#fd=open("certficado.out","wb")
#fd.write(res["bCertificate"])
#fd.close()
