def encode_name(value):
res = []
plain_flag = 1
if type(value) != typesx.StringType:
value = str(value)
for ch in value:
if ch in string_quoted:
res.append("\\" + ch)
else:
res.append(ch)
if not ch in plain_name_characters:
plain_flag = 0
if plain_flag:
return str.join("", res)
else:
return '"%s"' % str.join("", res)
def end(self):
m = self.whitespace.match(self.string, self.pos)
if m:
doc = self.docstring.findall(self.string[m.start():m.end()])
if len(doc):
self.current_doc += str.join("\n", doc)
self.pos = m.end()
return self.pos >= len(self.string)
def _dp_add_vlan(vid_dp, dp, vlan, logname):
if vlan.vid not in vid_dp:
vid_dp[vlan.vid] = set()
if len(vid_dp[vlan.vid]) > 1:
assert not vlan.bgp_routerid, \
"DPs {0} sharing a BGP speaker VLAN is unsupported".format(
str.join(", ", vid_dp[vlan.vid]),
)
if vlan not in dp.vlans:
dp.add_vlan(vlan)
vid_dp[vlan.vid].add(dp.name)
def exception_traceback(limit=50):
u"""
@return A string representation in typical Python format of the
currently active/raised exception.
"""
type, value, tb = sys.exc_info() # pylint: disable=redefined-builtin
lines = traceback.format_tb(tb, limit)
lines.extend(traceback.format_exception_only(type, value))
msg = u"Traceback (innermost last):\n"
if sys.version_info.major >= 3:
msg = msg + u"%-20s %s" % (str.join(u"", lines[:-1]), lines[-1])
else:
msg = msg + u"%-20s %s" % (string.join(lines[:-1], u""), lines[-1])
if sys.version_info.major < 3:
return msg.decode(u'unicode-escape', u'replace')
return msg
def get(self, expected):
m = self.whitespace.match(self.string, self.pos)
if m:
doc = self.docstring.findall(self.string[m.start():m.end()])
if len(doc):
self.current_doc += str.join("\n", doc)
self.pos = m.end()
pattern = self.patterns.get(expected)
if pattern:
m = pattern.match(self.string, self.pos)
if m:
self.pos = m.end()
return m.group(0)
else:
m = self.keyword_pattern.match(self.string, self.pos)
if m and m.group(0) == expected:
self.pos = m.end()
return True
def Convert_VCF_file_to_TAB_user_Kantale(inputFilename=None,
printHeader=False,
outputFilename=False,
expandInfo=False,
expandSample=False,
printFormat=False,
functionsToApplyToFields=None,
subtituteHeaderNames=None,
verbose=False):
""" """
#return dictionay with various statistics. To be extended..
ret = {}
ret["warnings"] = []
inputFile = open(inputFilename)
doubleLabelMessagePrinted = False
if expandInfo or expandSample:
if verbose:
print("Finding info and sample fields")
parsedComments = False
infoColumn = None
infoFields = set()
sampleColumn = None
sampleFields = set()
lineCounter = 0
while True:
line = inputFile.readline()
if line == "":
break
lineCounter += 1
if verbose:
if lineCounter % 10000 == 0:
print("Lines parsed: ", lineCounter)
lineSplitted = line.replace("\n", "").split()
if not parsedComments:
if lineSplitted[0] != "#CHROM":
continue
else:
parsedComments = True
infoColumn = lineSplitted.index("INFO")
sampleColumn = lineSplitted.index("FORMAT")
continue
infoFields = infoFields | set(
[x.split('=')[0] for x in lineSplitted[infoColumn].split(';')])
sampleFields = sampleFields | set(
lineSplitted[sampleColumn].split(':'))
if verbose:
print("Done finding info and sample fields")
infoFields = list(infoFields)
infoFields.sort()
# Move fields to the end
to_the_end = ['AC', 'MQ', 'QD', 'DP']
for value in to_the_end:
infoFields.append(infoFields.pop(infoFields.index(value)))
sampleFields = list(sampleFields)
inputFile.close()
inputFile = open(inputFilename)
headerSplitted = None
if outputFilename:
outputFile = open(outputFilename, "w")
headerPrinted = False
totalLines = lineCounter
lineCounter = 0
while True:
line = inputFile.readline()
if line == "":
break
if line[0] == "#" and line[1] == "#":
continue
lineCounter += 1
if verbose:
if lineCounter % 10000 == 0:
print("Lines parsed: " + str(lineCounter) + "/" +
str(totalLines))
lineSplitted = line.replace("\n", "").split("\t")
if not headerSplitted:
headerSplitted = list(lineSplitted)
headerToPrint = lineSplitted
continue
toPrint = {}
header_format = None
reference = None
alternative = None
for index, headerValue in enumerate(headerSplitted):
if headerValue.strip() == "":
continue
if headerValue == "#CHROM":
toPrint["#CHROM"] = lineSplitted[index]
elif headerValue == "POS":
toPrint["POS"] = lineSplitted[index]
elif headerValue == "ID":
toPrint["ID"] = "." if lineSplitted[
index] == "." else lineSplitted[index]
elif headerValue == "REF":
reference = lineSplitted[index]
toPrint["REF"] = reference
elif headerValue == "ALT":
alternative = lineSplitted[index]
toPrint["ALT"] = alternative
elif headerValue == "QUAL":
toPrint["QUAL"] = lineSplitted[index]
elif headerValue == "FILTER":
toPrint["FILTER"] = lineSplitted[index]
elif headerValue == "INFO":
if not expandInfo:
toPrint["INFO"] = lineSplitted[index]
else:
for infoField in lineSplitted[index].split(';'):
infoFieldSplitted = infoField.split('=')
#if len(infoFieldSplitted) == 1: toPrint[infoFieldSplitted[0]] = ""
if len(infoFieldSplitted) == 1:
toPrint[infoFieldSplitted[0]] = 'TRUE'
else:
toPrint[
infoFieldSplitted[0]] = infoFieldSplitted[1]
if not headerPrinted:
headerToPrint.insert(headerToPrint.index("INFO"),
infoFields)
headerToPrint.remove("INFO")
elif headerValue == "FORMAT":
header_format = lineSplitted[index]
if printFormat:
toPrint["FORMAT"] = header_format
else:
if not headerPrinted:
headerToPrint.remove("FORMAT")
else:
#We suggest that this is a sample
if not expandSample:
toPrint[headerValue] = lineSplitted[index]
else:
formatSplitted = header_format.split(':')
sampleSplitted = lineSplitted[index].split(':')
# Replace FORMAT NAMES to avoid duplicated entries
# by appending the sample name
for i, p in enumerate(formatSplitted):
formatSplitted[i] = p + '_' + headerValue
for formatIndex, formatValue in enumerate(formatSplitted):
#if formatValue == "GT":
# Genotype field
if formatValue.startswith('GT'):
if '/' in sampleSplitted[
formatIndex] or True: #This check is only there for future developement.
if '/' in sampleSplitted[formatIndex]:
GTSplitted = sampleSplitted[
formatIndex].split('/')
elif '|' in sampleSplitted[formatIndex]:
GTSplitted = sampleSplitted[
formatIndex].split('|')
else:
raise Exception(
"Genotype delimiter not recognized")
Alleles = []
for GT in GTSplitted:
if GT == ".":
Genotype = "."
elif GT == "0":
Genotype = reference
else:
Genotype = alternative[int(GT) - 1]
Alleles += [Genotype]
#if len(Alleles) == 2: toPrint["GT"] = Alleles[0] + "/" + Alleles[1]
#elif len(Alleles) == 1: toPrint["GT"] = Alleles[0] + "/" + Alleles[0]
if len(Alleles) == 2:
toPrint[formatValue] = Alleles[
0] + "/" + Alleles[1]
elif len(Alleles) == 1:
toPrint[formatValue] = Alleles[
0] + "/" + Alleles[0]
else:
raise Exception("Format error..")
else:
raise Exception(
"Not implemented.."
) #Please do someone.. (right now unreachable code)
else:
if formatValue in toPrint:
#This is indicative that something wrong happened.
if not doubleLabelMessagePrinted:
warningStr = "WARNING: field " + formatValue + " exists twice. This message is printed only once"
ret["warnings"] += [warningStr]
print(warningStr)
doubleLabelMessagePrinted = True
toPrint[formatValue +
"2"] = sampleSplitted[formatIndex]
else:
# If missing, add '.'
if sampleSplitted[0] == './.' or sampleSplitted[
0] == '.|.':
toPrint[formatValue] = '.'
else:
toPrint[formatValue] = sampleSplitted[
formatIndex]
if not headerPrinted:
# Append sample name to sampleFields
sampleFieldsWithName = [
el + '_' + headerValue for el in sampleFields
]
headerToPrint.insert(index, sampleFieldsWithName)
headerToPrint.remove(headerValue)
if not headerPrinted:
headerPrinted = True
headerToPrint = flatList(headerToPrint) #Make the list flat
if subtituteHeaderNames:
for substituteHeaderName in subtituteHeaderNames:
headerToPrint[headerToPrint.index(
substituteHeaderName[0])] = substituteHeaderName[1]
ret["header"] = headerToPrint
if printHeader:
headerToPrintStr = str.join('\t', headerToPrint)
if outputFilename:
outputFile.write(headerToPrintStr + "\n")
else:
print(headerToPrintStr)
toPrintNormalized = []
alreadyPrinted = []
for fieldToPrint in headerToPrint:
if fieldToPrint.strip() == "":
continue
if fieldToPrint in alreadyPrinted:
fieldToPrint = fieldToPrint + "2"
fieldAppliedFunction = toPrint[
fieldToPrint] if fieldToPrint in toPrint else ""
if functionsToApplyToFields:
if fieldToPrint in functionsToApplyToFields:
fieldAppliedFunction = functionsToApplyToFields[
fieldToPrint](fieldAppliedFunction)
toPrintNormalized += [fieldAppliedFunction]
alreadyPrinted += [fieldToPrint]
toPrintStr = str.join('\t', toPrintNormalized)
if outputFilename:
outputFile.write(toPrintStr + "\n")
else:
print(toPrintStr)
inputFile.close()
if outputFilename:
outputFile.close()
return ret
def execute(self,
stream,
process_limits=None,
origin_resource=None,
channels=None,
**kwargs):
''' Execute the stack node.
Parameters
----------
stream : :class:`obspy.core.Stream`
The data to process.
'''
# Check for needed keyword arguments.
if not self.kwargs_exists(['event'], **kwargs):
raise RuntimeError(
"The needed event argument was not passed to the execute method."
)
event = kwargs['event']
# Load the quarry_blast information.
blast_filename = self.pref_manager.get_value('blast_file')
if os.path.exists(blast_filename):
with open(blast_filename, 'r') as fp:
quarry_blast = json.load(
fp=fp, cls=quarry_blast_validation.QuarryFileDecoder)
else:
raise RuntimeError("Couldn't open the blast file %s.",
blast_filename)
# Get the baumit quarry blast id from the event tags.
id_tag = [x for x in event.tags if x.startswith('baumit_id:')]
if id_tag:
id_tag = id_tag[0]
else:
self.logger.error(
"No baumit quarry blast id tag found for event %d.",
event.db_id)
return
spec, baumit_id = id_tag.split(':')
# Compute the resultant of the stations.
res_stream = obspy.core.Stream()
res3d_stream = obspy.core.Stream()
orig_stream = obspy.core.Stream()
res_stations = []
for cur_detection in event.detections:
# TODO: Select the detection timespan only.
cur_stream = stream.select(network=cur_detection.scnl[2],
station=cur_detection.scnl[0],
location=cur_detection.scnl[3])
resultant_channels = ['Hnormal', 'Hparallel']
resultant_3d_channels = ['Hnormal', 'Hparallel', 'Z']
# Compute the 2D-resultant used for the magnitude computation.
cur_res_stream = self.compute_resultant(cur_stream,
resultant_channels)
if not cur_res_stream:
continue
#Compute the 3D-resultant used for reporting of the DUBA stations.
cur_res3d_stream = self.compute_resultant(cur_stream,
resultant_3d_channels)
orig_stream = orig_stream + cur_stream
res_stream = res_stream + cur_res_stream
res3d_stream = res3d_stream + cur_res3d_stream
res_stations.append(cur_detection.channel.parent_station)
# Handle the DUBAM station separately. Due to a time-error relative to
# DUBA, the DUBAM detections are most likely not included in the event
# detections.
stat_dubam = [x for x in res_stations if x.name == 'DUBAM']
self.logger.debug(
"##################### DUBAM HANDLING ###############")
self.logger.debug(stat_dubam)
if not stat_dubam:
cur_stream = stream.select(network='MSSNet',
station='DUBAM',
location='00')
self.logger.debug(cur_stream)
# Compute the 2D-resultant used for the magnitude computation.
resultant_channels = ['Hnormal', 'Hparallel']
cur_res_stream = self.compute_resultant(cur_stream,
resultant_channels)
if cur_res_stream:
#Compute the 3D-resultant used for reporting of the DUBA stations.
resultant_3d_channels = ['Hnormal', 'Hparallel', 'Z']
cur_res3d_stream = self.compute_resultant(
cur_stream, resultant_3d_channels)
orig_stream = orig_stream + cur_stream
res_stream = res_stream + cur_res_stream
res3d_stream = res3d_stream + cur_res3d_stream
cur_station = self.project.geometry_inventory.get_station(
name='DUBAM')[0]
res_stations.append(cur_station)
self.logger.debug("res_stations: %s", res_stations)
self.logger.debug("res_stream: %s", res_stream)
self.logger.debug("res3d_stream: %s", res3d_stream)
# Handle possible coordinate changes of station DUBAM.
# If the field x_dubam_1 is set, the alternative DUBAM position is
# used.
stat_dubam = [x for x in res_stations if x.name == 'DUBAM']
if stat_dubam:
stat_dubam = stat_dubam[0]
if quarry_blast[baumit_id]['x_dubam_1'] is not None:
stat_dubam.x = quarry_blast[baumit_id]['x_dubam_1']
stat_dubam.y = quarry_blast[baumit_id]['y_dubam_1']
stat_dubam.z = quarry_blast[baumit_id]['z_dubam_1']
stat_dubam.coord_system = 'epsg:31256'
# TODO: Check if all expected stations have got a trigger. Test the
# missing stations for available data an eventually missed events. Use
# a threshold value of the maximum amplitude as a rule whether to
# include the missed station into the PGV computation or not.
# Compute the max. PGV.
max_pgv = [
(str.join(':',
(x.stats.station, x.stats.network, x.stats.location)),
np.max(x.data)) for x in res_stream
]
max_pgv_3d = [
(str.join(':',
(x.stats.station, x.stats.network, x.stats.location)),
np.max(x.data)) for x in res3d_stream
]
# Compute the magnitude.
# TODO: Check the standard for the sign of the station correction.
# Brueckl subtracts the station correction.
proj_baumit = pyproj.Proj(init='epsg:' +
quarry_blast[baumit_id]['epsg'])
proj_wgs84 = pyproj.Proj(init='epsg:4326')
if res_stations:
stat_lonlat = np.array([x.get_lon_lat() for x in res_stations])
stat_x, stat_y = pyproj.transform(proj_wgs84, proj_baumit,
stat_lonlat[:, 0],
stat_lonlat[:, 1])
stat_z = np.array([x.z for x in res_stations])
quarry_x = quarry_blast[baumit_id]['x']
quarry_y = quarry_blast[baumit_id]['y']
quarry_z = quarry_blast[baumit_id]['z']
hypo_dist = np.sqrt((stat_x - quarry_x)**2 +
(stat_y - quarry_y)**2 +
(stat_z - quarry_z)**2)
stat_corr = np.zeros(len(max_pgv))
magnitude = np.log10([
x[1] * 1000 for x in max_pgv
]) + 1.6 * np.log10(hypo_dist) - 2.074 + stat_corr
else:
hypo_dist = []
magnitude = []
# Compute the PSD.
psd_data = {}
for cur_trace in orig_stream:
cur_psd_data = self.compute_psd(cur_trace)
cur_scnl = util.traceid_to_scnl(cur_trace.id)
cur_scnl_string = ':'.join(cur_scnl)
psd_data[cur_trace.id] = cur_psd_data
# Compute the dominant frequency.
dom_frequ = {}
dom_stat_frequ = {}
for cur_station in res_stations:
cur_psd_keys = [
x for x in psd_data.keys()
if x.startswith(cur_station.network + '.' + cur_station.name +
'.')
]
cur_df = []
for cur_key in cur_psd_keys:
cur_nfft = psd_data[cur_key]['n_fft']
left_fft = int(np.ceil(cur_nfft / 2.))
max_ind = np.argmax(psd_data[cur_key]['psd'][1:left_fft])
dom_frequ[cur_key] = psd_data[cur_key]['frequ'][max_ind]
cur_df.append(dom_frequ[cur_key])
dom_stat_frequ[cur_station.snl_string] = np.mean(cur_df)
# Update the quarry blast information dictionary.
export_max_pgv = dict(max_pgv)
export_magnitude = dict(
list(zip([x.snl_string for x in res_stations], magnitude)))
quarry_blast[baumit_id]['computed_on'] = utcdatetime.UTCDateTime()
quarry_blast[baumit_id]['event_time'] = event.start_time
quarry_blast[baumit_id]['max_pgv'] = {}
quarry_blast[baumit_id]['max_pgv']['data'] = export_max_pgv
quarry_blast[baumit_id]['max_pgv'][
'used_channels'] = resultant_channels
quarry_blast[baumit_id]['max_pgv_3d'] = {}
quarry_blast[baumit_id]['max_pgv_3d']['data'] = dict(max_pgv_3d)
quarry_blast[baumit_id]['max_pgv_3d'][
'used_channels'] = resultant_3d_channels
quarry_blast[baumit_id]['magnitude'] = {}
quarry_blast[baumit_id]['magnitude']['station_mag'] = export_magnitude
quarry_blast[baumit_id]['magnitude']['network_mag'] = np.mean(
magnitude)
quarry_blast[baumit_id]['magnitude']['network_mag_std'] = np.std(
magnitude)
quarry_blast[baumit_id]['dom_frequ'] = dom_frequ
quarry_blast[baumit_id]['dom_stat_frequ'] = dom_stat_frequ
quarry_blast[baumit_id]['hypo_dist'] = dict(
list(zip([x.snl_string for x in res_stations], hypo_dist)))
# Save the quarry blast information.
with open(blast_filename, 'w') as fp:
json.dump(quarry_blast,
fp=fp,
cls=quarry_blast_validation.QuarryFileEncoder)
# Write a result file for the event.
output_dir = self.pref_manager.get_value('report_data_dir')
filename = 'blast_report_data_event_%010d.pkl' % event.db_id
report_data = {}
report_data['baumit_id'] = baumit_id
report_data['blast_data'] = quarry_blast[baumit_id]
report_data['psd_data'] = psd_data
with open(os.path.join(output_dir, filename), 'w') as fp:
pickle.dump(report_data, fp)
# Clear the computation request flag in the event database.
event.tags.remove('mss_result_needed')
event.tags.append('mss_result_computed')
event.write_to_database(self.project)