Beispiel #1
0
    y = x - data.field('cmodelmag_r')[ind]
    pl.plot(x, y, 'g.', alpha=0.3, mec='none')
    pl.plot([14, 24], [0.145, 0.145], 'k', alpha=0.2)
    pl.xlabel('r psfmag')
    pl.ylabel('r psfmag - r cmodelmag')
    pl.ylim(-0.2, 0.4)
    f.savefig('../plots/foo.png')

    assert 0
    print '\n\n\nThese are the column names:\n%s\n\n' % names

    Ng = np.where(data.field('type') == 3)[0].size
    Ns = np.where(data.field('type') == 6)[0].size
    print '\nSDSS says there are %d galaxies and %d stars.\n\n' % (Ng, Ns)

    photo, morph, scalers, ind = prepare_data(data)

    photo = scalers[0].inverse_transform(photo)
    morph = scalers[1].inverse_transform(morph)

    labels = ['u', 'g', 'r', 'i', 'z']
    kind = data.field('type')[ind]

    mags = photo[:, :5]
    mags[:, :2] += mags[:, 2, None]
    mags[:, 3:] += mags[:, 2, None]
    plot_morph(mags, morph[:, -5:], kind, labels, '../plots/petro50_1.png', 'PetroR50')
    
    f = pl.figure()
    ind = kind == 6
    pl.plot(mags[ind, 2], photo[ind, 7], 'g.', alpha=0.3, mec='none')
Beispiel #2
0
def Dutch_idea_density(inputfile, output='PID'):
	"""
	This functions takes as input a Frog-file.
	The propositional idea density and baseline score are based upon
	the POS-tag of each word.
	All words that meet the baseline or propositional conditions are counted.
	By default, the function returns the propositional density of the textfile.
	"""

	text = prepare_data(inputfile)
	result_dict = {'overall': 0, 'per_sentence': []}

	# Initiate counts.
	wordcount = 0
	baseline_count = 0
	total_baseline = 0
	count = 0
	total_propcount = 0
	total_wordcount = 0

	for sent_id, sentence in enumerate(text):
		# The amount of words is limited to the smallest text.
		if wordcount < 12030:

			# Word count and baseline count.
			for word in sentence:
				if word[1] != 'LET()':
					# Count each token, except punctuation.
					wordcount += 1
				if word[1][:2] in baseline:
					# Count all tags that are part of the Baseline list.
					baseline_count += 1

			# Create sentence string of all tokens.
			sent = [word[0] for word in sentence]
			remove1 = re.compile(r"([\"\']\, [\"\'])|(\[[\"\'])|([\'\"]\])")
			sent_without = remove1.sub(" ", str(sent))

			# Create tag string of the 2 first characters of each tag.
			tag = [word[1][:2] for word in sentence]
			tag_without = remove1.sub(" ", str(tag))

			# Define the value of each token variable (when present).
			# & Create a string of all preceding tags
			# and a string of all following tags and tokens.
			for i in range(0, len(sentence)):
				vari_last = sentence[-2]
				try:
					vari_min4 = sentence[i - 4]
					tag_prev = str(vari_min4[1][:2])
				except IndexError:
					tag_prev = str(0)
					pass
				try:
					vari_min3 = sentence[i - 3]
					tag_prev = str(tag_prev + ' ' + vari_min3[1][:2])
				except IndexError:
					pass
				try:
					vari_min2 = sentence[i - 2]
					tag_prev = str(tag_prev + ' ' + vari_min2[1][:2])
				except IndexError:
					pass
				vari_min1 = sentence[i - 1]
				tag_prev = str(tag_prev + ' ' + vari_min1[1][:2])
				vari = sentence[i]
				try:
					vari1 = sentence[i + 1]
					sent_next = str(vari1[0])
					tag_next = str(vari1[1][:2])
				except IndexError:
					pass
				try:
					vari2 = sentence[i + 2]
					sent_next = str(sent_next + ' ' + vari2[0])
					tag_next = str(tag_next + ' ' + vari2[1][:2])
				except IndexError:
					pass
				try:
					vari3 = sentence[i + 3]
					sent_next = str(sent_next + ' ' + vari3[0])
					tag_next = str(tag_next + ' ' + vari3[1][:2])
				except IndexError:
					pass
				try:
					vari4 = sentence[i + 4]
					sent_next = str(sent_next + ' ' + vari4[0])
					tag_next = str(tag_next + ' ' + vari4[1][:2])
				except IndexError:
					pass
				try:
					vari5 = sentence[i + 5]
					sent_next = str(sent_next + ' ' + vari5[0])
					tag_next = str(tag_next + ' ' + vari5[1][:2])
				except IndexError:
					pass
				try:
					vari6 = sentence[i + 6]
					sent_next = str(sent_next + ' ' + vari6[0])
					tag_next = str(tag_next + ' ' + vari6[1][:2])
				except IndexError:
					pass
				try:
					vari7 = sentence[i + 7]
					sent_next = str(sent_next + ' ' + vari7[0])
					tag_next = str(tag_next + ' ' + vari7[1][:2])
				except IndexError:
					pass
				try:
					vari8 = sentence[i + 8]
					sent_next = str(sent_next + ' ' + vari8[0])
					tag_next = str(tag_next + ' ' + vari8[1][:2])
				except IndexError:
					pass
				try:
					vari9 = sentence[i + 9]
					sent_next = str(sent_next + ' ' + vari9[0])
					tag_next = str(tag_next + ' ' + vari9[1][:2])
				except IndexError:
					pass
				try:
					vari10 = sentence[i + 10]
					sent_next = str(sent_next + ' ' + vari10[0])
					tag_next = str(tag_next + ' ' + vari10[1][:2])
				except IndexError:
					pass
				try:
					vari11 = sentence[i + 11]
					sent_next = str(sent_next + ' ' + vari11[0])
					tag_next = str(tag_next + ' ' + vari11[1][:2])
				except IndexError:
					pass

				# Count propositions.
				# Simple proposition count: always/never.
				if vari[1].startswith(never_count):
					continue
				elif vari[1].startswith(always_count):
					# Never count existential 'Er'.
					if vari[0] not in ('er', 'Er'):
						count += 1

				# Proposition count restriction rules:
				elif vari[1][:1] == 'N':
					if vari[1].endswith('gen)'):  # Only count [noun]+'s.
						if not vari_min1[0] in ("'s", 'des'):
							count += 1

				elif vari[1][:2] == 'AD':
					if not vari[1].startswith('ADJ(nom'):
						# Never count nominative adjectives.
						count += 1

				elif vari[1][:2] == 'TW':
					if not re.search(r'TW\(((hoofd)|(rang)),nom', vari[1]):
						# Never count nominative numerals.
						count += 1

				elif vari[1].startswith('VNW(onbep,pron,stan'):
					if vari[0] == 'wat':
						# Only count 'wat'.
						count += 1

				elif vari[1].startswith('VNW(onbep'):
					if vari[0] in ('Geen', 'geen'):
						if not re.search(r'\[Gg]een .* tenzij ', sent_without):
							# 'Geen ... tenzij' is one proposition, only count 'tenzij'.
							count += 1
					elif vari[0] == 'enkele':
						if not re.search(r'\b[Gg]een enkele ', sent_without):
							# 'Geen ... enkele' is one proposition, only count 'geen'.
							count += 1
					elif vari[0] == 'meer':
						if not re.search(r'\b[Gg]een (\w+ )*meer', sent_without):
							# 'Geen ... meer' is one proposition, only count 'geen'.
							count += 1
					else:
						count += 1

				# Empty expression, preceding verbs (~'to'[+verb]).
				elif vari[1] == 'VZ(init)':
					if vari[0] in ('Te', 'te'):
						if vari1[1][:2] != 'WW':
							# Never count 'te' [+ WW].
							count += 1
					elif vari[0] in ('Om', 'om'):
						if not re.search(r'\bom (\w+ )*te ', sent_without):
							# Never count 'om' [+ 'te' WW].
							count += 1
					elif vari[0] in ('Aan', 'aan'):
						if not ((vari1[0] == 'het' or vari1[0] == "'t") and
								vari2[1][:2] == 'WW'):
							# Never count 'aan' [+ 'het' WW].
							count += 1
					else:
						count += 1

				# Wordcombinations that count as 1 proposition:
				elif vari[1] == 'BW()':
					if vari[0] in ('zowel', 'Zowel'):
						if not re.search(r'\b[Zz]owel .* als ', sent_without):
							# 'Zowel ... als' is one proposition, only count 'als'.
							count += 1
					elif vari[0] in ('nog', 'Nog'):
						if not re.search(r'\b[Nn]og nooit', sent_without):
							# 'Nog nooit' is one proposition, only count 'nooit'.
							count += 1
					elif vari[0] == 'niet':
						if not re.search(r'\bniet.*((meer)|([Tt]enzij))', sent_without):
							# 'Niet ... meer/tenzij' is one proposition, only count 'meer'/'tenzij'.
							count += 1
					elif vari[0] in ('ook', 'Ook'):
						if not vari1[0] == 'al':
							# 'Ook al' is one proposition, only count 'al'.
							count += 1
					else:
						count += 1

				elif vari[1][:2] == 'VG':
					if vari[0] == 'dan':
						if not re.search(r'\b(([Mm]eer)|([Aa]ls)).*dan ', sent_without):
							# 'Meer/Als ... dan' is one proposition, only count 'Meer'/'Als'.
							count += 1
					else:
						count += 1

				elif vari[1].startswith('WW'):

					# Auxiliary verbs: modal verbs and copula verbs.
					if re.search(aux, vari[2]):

						# 'Hebben'
                        # Always count lexical verb.
                        # Never count all other instances.
						if re.search(r'\[heb\]', vari[2]):
							if (vari_min2[0] == 'te' and vari_min1[1][:4] == 'WW(i'):
								continue  # Never count.
							elif (vari_min2[1][:4] == 'WW(i'and vari_min1[0] == 'te'):
								continue  # Never count.
							elif vari[0] == 'gehad':
								if vari1[1][:4] != 'WW(i':
									# 'Gehad' always functions as lexical verb.
									count += 1
							else:
								if not (vari_min1[1][:4] == 'WW(v' or vari1[1][:4] == 'WW(v'):
									if not (vari_min2[1][:4] == 'WW(v' and vari_min1[1][:2] not in ('VG', 'LE')):
										if (vari1[0] == ',' or vari1[1][:2] == 'VG'):
											count += 1  # Lexical verb.
										else:
											if not re.search(r'WW ((VZ|LI|N\(|VN|SP|TW|AD|BW) )*WW', tag_without):
												# When only verb in sentence: lexical verb.
												count += 1
											elif not (re.search(r'\[heb\].*WW\((vd|inf)\,vrij', str(sentence)) or
													  re.search(r'WW\((vd|inf)\,vrij.*\[heb\]', str(sentence))):
												if (re.search(r'\[heb\].*VZ\(fin\)', str(sentence)) or
														re.search(r'VZ\(fin\).*\[heb\]', str(sentence))):
													if (re.search(door_aan, str(vari1)) or re.search(door_aan, str(vari2)) or
															re.search(door_aan, str(vari3)) or re.search(door_aan, str(vari4))):
														# Always count lexical verb with fixed preposition.
														count += 1
												else:
													# When other verb functions as ADJ: Lexical verb.
													count += 1
											else:
												if vari_min1[2].startswith(modal_list):
													# When preceded by modal verb: Lexical verb.
													count += 1

						# 'Zijn' & 'Worden':
						# Always count copula verb, accompanied by Noun phrase.
						# Never count all other instances.
						elif re.search(r'word', vari[2]) or re.search(r'zijn', vari[2]):
							if (vari_min2[0] == 'te' and vari_min1[1][:4] == 'WW(i'):
								continue  # Never count.
							elif (vari_min2[1][:4] in ('WW(v', 'ADJ(') and vari_min1[0] == 'te'):
								continue  # Never count.
							elif not (re.search(r'(\w+ )*aan het ', sent_next) and
									  re.search(r'VZ LI WW', tag_next)):
								if vari_min1[1][:10] != 'WW(vd,vrij' and vari1[1][:10] != 'WW(vd,vrij':
									if not re.search(r'ADJ\([vp]', vari_min1[1][:5]):
										if vari_last != vari:
											if vari1[1][:3] == 'ADJ':
												if vari2[1] != 'LET()':
													if not re.search(hulp, tag_next):
														if re.search(kop, tag_next):
															count += 1
											else:
												if re.search(aux, vari_min1[2]):
													if vari1[1][:4] != 'WW(v':
														if not re.search(hoe, sent_without):
															if (vari_min2[1][:1] == 'N' or vari_min2[1][:5] == 'ADJ(n'):
																if not vari_min3[1][:2] == 'VZ':
																	count += 1
															elif vari_min2[1][:5] not in ('ADJ(v', 'ADJ(p', 'WW(vd'):
																if (vari_min3[1][:1] == 'N' or vari_min3[1][:5] == 'ADJ(n'):
																	count += 1
												elif not vari1[1][:2] == 'VZ':
													if (vari_min1[1][:2] == 'VZ' and vari[1][:4] == 'WW(i'):
														if (vari_min2[1][:1] == 'N' or vari_min2[1][:5] == 'ADJ(n'):
															count += 1
													else:
														if vari1[1] == 'LET()':
															if (vari_min1[1][:1] in ('N', 'S') or vari_min1[1][:5] == 'ADJ(n'):
																if not (vari_min2[1][:2] == 'VZ' or
																		(vari_min3[1][:2] == 'VZ' and vari_min2[1][:2] in ('LI', 'VN'))):
																	count += 1
														elif vari_min1[1][:3] == 'VNW':
															if (vari1[1][:2] in ('N(', 'LI', 'SP', 'VN') and vari3[1][:4] != 'WW(v'):
																count += 1
															elif (vari2[1][:2] in ('N(', 'SP') and
																	vari3[1][:4] != 'WW(v' and vari4[1][:4] != 'WW(v'):
																count += 1
															elif (vari1[1][:2] == 'BW' and vari2[1][:2] in ('N(', 'LI', 'SP') and
																	vari4[1][:4] != 'WW(v' and vari5[1][:4] != 'WW(v'):
																count += 1
															elif (vari1[1][:2] == 'BW' and vari2[1][:2] == 'VN' and
																	(vari3[1][:1] == 'N' or vari4[1][:1] == 'N') and
																	vari4[1][:4] not in ('WW(v', 'WW(i', 'ADJ(') and
																	vari5[1][:4] not in ('WW(v', 'WW(i', 'ADJ(')):
																count += 1
														elif vari1[1][:2] == 'LI':
															if (vari_min1[1][:5] not in ('WW(vd', 'ADJ(p', 'ADJ(v') and
																	vari3[1][:5] not in ('WW(vd', 'ADJ(p', 'ADJ(v') and
																	vari4[1][:5] not in ('WW(vd', 'ADJ(p', 'ADJ(v') and
																	vari5[1][:5] not in ('WW(vd', 'ADJ(p', 'ADJ(v') and
																	vari6[1][:5] not in ('WW(vd', 'ADJ(p', 'ADJ(v')):
																if not (vari4[1][:2] == 'VZ' or vari5[1][:2] == 'VZ' or
																		vari6[1][:2] == 'VZ'):
																	count += 1
																elif vari4[0] == 'van':
																	count += 1
														elif (vari1[1][:2] in ('SP', 'VN') and vari2[1][:2] == 'LI'):
															if (vari4[1][:5] not in ('WW(vd', 'ADJ(p', 'ADJ(v') and
																	vari5[1][:5] not in ('WW(vd', 'ADJ(p', 'ADJ(v')):
																count += 1
														elif vari2[1] == 'LET()':
															if vari1[1][:1] in ('N', 'S'):
																count += 1
										else:
											if (vari_min1[1][:2] != 'WW' and vari_min1[1][:2] != 'VZ'):
												if (vari_min1[1][:2] in ('N(', 'AD') and  # Adjective is nominative.
														not re.search(r'VZ (LI )?(AD )?', tag_prev)):
													count += 1
											else:
												if re.search(aux, vari_min1[2]):
													if (vari_min2[1][:1] == 'N' or vari_min2[1][:5] == 'ADJ(n'):
														count += 1
													else:
														if (vari_min2[1][:2] == 'VZ' and vari_min3[1][:2] in ('N(', 'SP')):
															count += 1
												else:
													if (vari_min1[1][:2] == 'VZ' and vari_min2[1][:2] in ('N(', 'SP')):
														count += 1

						# Modal verbs.
                        # Always count lexical verb.
                        # Never count all other instances.
						elif re.search(modal, vari[2]):
							if not (re.search(r'WW', tag_next) or re.search(r'WW', tag_prev)):
								# Lexical verb is missing, modal verb = replacement proposition.
								count += 1
							elif not (vari[2][:4] == '[zal' or vari1[1][:2] == 'WW' or vari_min1[1][:2] == 'WW'):
								if re.search(r'\[ga', vari[2]):
									if vari_min1[0] == 'te':
										if vari_min2[1][:4] != 'WW(i':
											count += 1
									elif not ((vari2[1][:4] == 'WW(i' and vari1[1][:2] not in ('LE', 'VG')) or
											  (vari3[1][:4] == 'WW(i' and (vari1[1][:2] not in ('LE', 'VG') and
																		   vari2[:2] not in ('LE', 'VG'))) or
											  (vari4[1][:4] == 'WW(i' and (vari1[1][:2] not in ('LE', 'VG') and
																		   vari2[1][:2] not in ('LE', 'VG') and
																		   vari3[1][:2] not in ('LE', 'VG'))) or
											  (vari5[1][:4] == 'WW(i' and (vari1[1][:2] not in ('LE', 'VG') and
																		   vari2[1][:2] not in ('LE', 'VG') and
																		   vari3[1][:2] not in ('LE', 'VG') and
																		   vari4[1][:2] not in ('LE', 'VG'))) or
											  (vari6[1][:4] == 'WW(i' and (vari1[1][:2] not in ('LE', 'VG') and
																		   vari2[1][:2] not in ('LE', 'VG') and
																		   vari3[1][:2] not in ('LE', 'VG') and
																		   vari4[1][:2] not in ('LE', 'VG'))) or
											  (vari7[1][:4] == 'WW(i' and (vari1[1][:2] not in ('LE', 'VG') and
																		   vari2[1][:2] not in ('LE', 'VG') and
																		   vari3[1][:2] not in ('LE', 'VG') and
																		   vari4[1][:2] not in ('LE', 'VG')))):
										count += 1
								elif not re.search(mod1, tag_without):
									if vari2[1][:4] != 'WW(i':
										if vari_min1[1] == 'LET()':
											if not (re.search(mod2, tag_without) or vari1[1][:2] == 'LE'):
												# Matches long distance infinitives:
												if not (((vari3[1][:4] == 'WW(i' or vari4[1][:4] == 'WW(i') and
														 (vari2[1] != 'LET()' and vari3[1] != 'LET()')) or
														((vari5[1][:4] == 'WW(i' or vari6[1][:4] == 'WW(i') and
														 (vari2[1] != 'LET()' and vari3[1] != 'LET()' and
														  vari4[1] != 'LET()' and vari5[1]) != 'LET()')):
													count += 1
										elif vari1[1] == 'LET()':
											if vari_min1[1][:2] == 'VN':
												count += 1
										elif (vari1[1][:2] == 'VG' and vari1[0] == 'dat'):
											count += 1

						# Secondary copula.
						# Always count sec. copula, accompanied by Noun phrase.
						# Always count lexical verbs.
						# Never count all other instances.
						elif re.search(sec_cop, vari[2]):
							if re.search(r'blijf', vari[2]):
								# Never count auxiliary verbs:
								if not ((vari1[1][:4] in ('WW(i', 'WW(v') or vari_min1[1].startswith('WW(i')) or
										(vari2[1][:4] in ('WW(i', 'WW(v') and vari1[1][:2] not in ('LE', 'VG')) or
										(vari3[1][:4] in ('WW(i', 'WW(v') and (vari1[1][:2] not in ('LE', 'VG') and
																			   vari2[:2] not in ('LE', 'VG'))) or
										(vari4[1][:4] in ('WW(i', 'WW(v') and (vari1[1][:2] not in ('LE', 'VG') and
																			   vari2[1][:2] not in ('LE', 'VG') and
																			   vari3[1][:2] not in ('LE', 'VG'))) or
										(vari5[1][:4] in ('WW(i', 'WW(v') and (vari1[1][:2] not in ('LE', 'VG') and
																			   vari2[1][:2] not in ('LE', 'VG') and
																			   vari3[1][:2] not in ('LE', 'VG') and
																			   vari4[1][:2] not in ('LE', 'VG'))) or
										(vari6[1][:4] in ('WW(i', 'WW(v') and (vari1[1][:2] not in ('LE', 'VG') and
																			   vari2[1][:2] not in ('LE', 'VG') and
																			   vari3[1][:2] not in ('LE', 'VG') and
																			   vari4[1][:2] not in ('LE', 'VG'))) or
										(vari7[1][:4] in ('WW(i', 'WW(v') and (vari1[1][:2] not in ('LE', 'VG') and
																			   vari2[1][:2] not in ('LE', 'VG') and
																			   vari3[1][:2] not in ('LE', 'VG') and
																			   vari4[1][:2] not in ('LE', 'VG')))):
									if vari_min1[0] in ('hier', 'daar', 'waar'):
										# Always count lexical verb (~zich bevinden).
										count += 1
									elif not (re.search(r'ADJ\([vp]', vari1[1]) and vari2[1][:4] != 'VZ(i'):
										if not (vari_min1[1][:5] in ('ADJ(v', 'ADJ(p', 'WW(vd') and
												not vari_min2[1][:2] in ('LI')):
											if vari1[0] in ('bij', 'in', 'hier'):
												# Always count lexical verb (~zich bevinden).
												count += 1
											elif vari_min1[1][:3] == 'ADJ':
												# Adjective functions as noun, verb is counted.
												count += 1
											else:
												if (vari_min1[0] == 'te' or re.search(modal, vari_min1[2])):
													if not vari_min2[1][:3] in ('WW(', 'ADJ'):
														count += 1
												elif re.search(r' (bij|in) (\w+ )+[Bb]l(ij|ee?)[fv](en)?t? ', sent_without):
													count += 1
												elif not re.search(r'[Bb]l(ij|ee?)[fv](en)?t? (\w+ )*(\w+n) \W', sent_without):
													if (vari1[1][:2] == 'LI' and vari3[1][:5] not in ('ADJ(v', 'ADJ(p')):
														count += 1
													elif re.search(r'[Bb]l(ij|ee?)[fv](en)?t? (\w+ )*(over|achter|af|uit|op) ', sent_without):
														count += 1
													elif (vari3[1].startswith('N') and vari4[1][:5] not in ('ADJ(v', 'ADJ(p')):
														count += 1
													else:
														if not (vari1[1] == 'BW' and vari2[1][:5] in ('ADJ(v', 'ADJ(p')):
															count += 1

							elif re.search(r'\[dunk\]', vari[2]):
								count = count  # Never count auxiliary verb 'dunken'.

							elif re.search(r'kom', vari[2]):
								# Matches any form of 'voorkomen'.
								if vari[0].startswith('voorkomend'):
									# Should be tagged as adjective.
									count += 1
								elif not (vari_min1[1][:7] == 'VNW(bez' or vari_min1[1][:2] == 'LI'):
									# Should be tagged as noun.
									if (vari_min1[0] == 'te' or re.search(modal, vari_min1[2])):
										count += 1  # Always count lexical verb.
							else:
								if (re.search(op, sent_without) and re.search(r'lijk', vari[2])):
									# Always count lexical verb 'lijken op'.
									count += 1
								elif (re.search(licht, sent_without) and re.search(r'schijn', vari[2])):
									# Always count lexical verb '[licht] schijnen'.
									count += 1
								elif (re.search(r'blijk', vari[2]) and
										(re.search(r' zag', sent_without) or re.search(bleek, sent_without))):
									# Frequently falsely tagged adjective 'bleek'.
									count += 1
								elif not re.search(kop_hulp, sent_next):
									if not re.search(alsof_dat, sent_without):
										if not re.search(kop_het, sent_without):
											if not (re.search(r'[Nn]aar ((he)|(\'))t schijnt', sent_without) and
													vari[0] == 'schijnt'):
												if vari1[1][:2] != 'VZ':
													if (vari_min1[1][:5] not in ('ADJ(v', 'ADJ(p', 'WW(vd')):
														if vari1[1][:5] in ('ADJ(v', 'ADJ(p'):
															if (vari2[1][:1] == 'N' or vari3[1][:1] == 'N'):
																count += 1
														elif vari2[1][:5] in ('ADJ(v', 'ADJ(p', 'WW(vd'):
															if vari1[1].startswith('LI'):
																count += 1
														elif (re.search(r'[ov]d,[np]', vari1[1][:7]) or
																re.search(r'[ov]d,[np]', vari2[1][:7]) or
																re.search(r'[ov]d,[np]', vari3[1][:7])):
															count += 1
														elif vari1[1].startswith('SP'):
															count += 1
														elif vari2[1].startswith('N') or vari2[1].startswith('SP'):
															count += 1
														elif vari3[1].startswith('N') or vari3[1].startswith('SP'):
															if not vari4[1].startswith('WW'):
																count += 1
														elif vari4[1].startswith('N'):
															count += 1

					
					# Lexical verbs.
					# Never count verbs that function as nouns.
					else:
						if re.search(r'WW.*\,nom\,', vari[1]):
							if vari_min1[0] in ("'t", 'het') and vari_min2[0] == 'aan':
								count += 1  # Falsely tagged lexical verbs.
						elif re.search(r'(^\[((begin)|(lig)|(sta)|(zit))\])', vari[2]):
							if not re.search(r'(^(\w+ )*te \w+n)', sent_next):
								# Never count replacement auxiliary verbs.
								count += 1
							elif re.search(r' ((om)|(dat)|(en)) (\w+ )*te \w+n', sent_next):
								if not re.search(r' te (\w+ )*te', sent_next):
									count += 1
						else:
							count += 1
		
		# Sentence PID
		sentence_pid = count/wordcount
		result_dict['per_sentence'].append(sentence_pid)
		
		# Add the proposition count of each sentence to the total proposition count.
		total_propcount += count
		total_wordcount += wordcount
		total_baseline += baseline_count
		baseline_count = 0
		wordcount = 0
		count = 0
	
	# Measure propositional idea density
	# by dividing the total proposition count by the total number of words.
	prop_density = total_propcount / total_wordcount
	result_dict['overall'] = prop_density
	
	# Measure the baseline score
	# by dividing the total baseline count by the total number of words.
	baseline_score = total_baseline / total_wordcount

	if output == 'PID':
		return result_dict # prop_density
	elif output == 'BL':
		return baseline_score
Beispiel #3
0
    seed = 1234567

    #f = '../../data/mr10k_fluxes_rfadely.fit'
    f = '../../data/mr10k_short_rfadely.fit'
    f = pf.open(f)
    data = f[1].data
    names = f[1].columns.names
    f.close()

    print '\n\n\nThese are the column names:\n%s\n\n' % names

    Ng = np.where(data.field('type') == 3)[0].size
    Ns = np.where(data.field('type') == 6)[0].size
    print '\nSDSS says there are %d galaxies and %d stars.\n\n' % (Ng, Ns)

    features = ['modelmags', 'psffwhms']
    targets = ['p50']
    mrng = np.array([15, 22])
    photo, morph, scalers, ind = prepare_data(data, features, ['p50'],
                                              color_range=None, scale=None,
                                              r_modelmag_range=mrng)
    
    #determine_parms(photo, morph, SVR(), seed, 'SVR', 'svr_grid.txt')

    rgrname = 'KNN'
    rgr = KNeighborsRegressor()
    title = 'Regression: %s, Features: (%s)' % (rgrname, ', '.join(features))
    #plotname = '../plots/p50_mf_r_%s_%0.2f-%0.2f.png' % (rgrname, mrng[0], mrng[1])
    plotname='../../plots/foo.png'
    plot_r_predictions(photo, morph, scalers, rgr, seed, title, plotname, single=False)