def process_dir(abbr, verbose, summary, settings):
person_filenames = glob.glob(
os.path.join(get_data_dir(abbr), 'people', '*.yml'))
retired_filenames = glob.glob(
os.path.join(get_data_dir(abbr), 'retired', '*.yml'))
org_filenames = glob.glob(
os.path.join(get_data_dir(abbr), 'organizations', '*.yml'))
validator = Validator(settings, abbr)
for filename in person_filenames:
print_filename = os.path.basename(filename)
with open(filename) as f:
person = yaml.load(f)
validator.validate_person(person, print_filename)
for filename in retired_filenames:
print_filename = os.path.basename(filename)
with open(filename) as f:
person = yaml.load(f)
validator.validate_person(person, print_filename, retired=True)
for filename in org_filenames:
print_filename = os.path.basename(filename)
with open(filename) as f:
org = yaml.load(f)
validator.validate_org(org, print_filename)
validator.print_validation_report(verbose)
if summary:
validator.print_summary()
def process_dir(abbr, verbose, municipal, date): # pragma: no cover
legislative_filenames = glob.glob(os.path.join(get_data_dir(abbr), "legislature", "*.yml"))
executive_filenames = glob.glob(os.path.join(get_data_dir(abbr), "executive", "*.yml"))
municipality_filenames = glob.glob(os.path.join(get_data_dir(abbr), "municipalities", "*.yml"))
retired_filenames = glob.glob(os.path.join(get_data_dir(abbr), "retired", "*.yml"))
settings_file = os.path.join(os.path.dirname(__file__), "../settings.yml")
with open(settings_file) as f:
settings = load_yaml(f)
try:
validator = Validator(abbr, settings)
except BadVacancy:
sys.exit(-1)
all_filenames = [
(PersonType.LEGISLATIVE, legislative_filenames),
(PersonType.RETIRED, retired_filenames),
(PersonType.EXECUTIVE, executive_filenames),
]
if municipal:
all_filenames.append((PersonType.MUNICIPAL, municipality_filenames))
for person_type, filenames in all_filenames:
for filename in filenames:
print_filename = os.path.basename(filename)
with open(filename) as f:
person = load_yaml(f)
validator.validate_person(person, print_filename, person_type, date)
error_count = validator.print_validation_report(verbose)
return error_count
def process_dir(abbr, verbose, summary, settings): # pragma: no cover
person_filenames = glob.glob(os.path.join(get_data_dir(abbr), "people", "*.yml"))
retired_filenames = glob.glob(os.path.join(get_data_dir(abbr), "retired", "*.yml"))
org_filenames = glob.glob(os.path.join(get_data_dir(abbr), "organizations", "*.yml"))
try:
validator = Validator(abbr, settings)
except BadVacancy:
sys.exit(-1)
for filename in person_filenames:
print_filename = os.path.basename(filename)
with open(filename) as f:
person = load_yaml(f)
validator.validate_person(person, print_filename)
for filename in retired_filenames:
print_filename = os.path.basename(filename)
with open(filename) as f:
person = load_yaml(f)
validator.validate_person(person, print_filename, retired=True)
for filename in org_filenames:
print_filename = os.path.basename(filename)
with open(filename) as f:
org = load_yaml(f)
validator.validate_org(org, print_filename)
error_count = validator.print_validation_report(verbose)
if summary:
validator.print_summary()
return error_count
def find_by_id(id):
id = id.split("/")[1]
choices = glob.glob(
os.path.join(get_data_dir("*"), "people", f"*-{id}.yml")) + glob.glob(
os.path.join(get_data_dir("*"), "retired", f"*-{id}.yml"))
if len(choices) != 1:
raise ValueError(f"unknown id {id}")
return choices[0]
def find_by_id(id):
id = id.split('/')[1]
choices = glob.glob(
os.path.join(get_data_dir('*'), 'people', f'*-{id}.yml')) + glob.glob(
os.path.join(get_data_dir("*"), "retired", f"*-{id}.yml"))
if len(choices) != 1:
raise ValueError(f'unknown id {id}')
return choices[0]
def entrypoint(incoming, old, new, keep, remove_identical, copy_new,
interactive):
"""
Script to assist with merging legislator files.
Can be used in two modes: incoming or file merge.
Incoming mode analyzes incoming/ directory files (generated with to_yaml.py)
and discovers identical & similar files to assist with merging.
File merge mode merges two legislator files.
"""
if incoming:
abbr = incoming
existing_people = []
for filename in (
glob.glob(os.path.join(get_data_dir(abbr), 'people/*.yml')) +
glob.glob(os.path.join(get_data_dir(abbr), 'retired/*.yml'))):
with open(filename) as f:
existing_people.append(load_yaml(f))
new_people = []
incoming_dir = get_data_dir(abbr).replace('data', 'incoming')
for filename in glob.glob(os.path.join(incoming_dir, 'people/*.yml')):
with open(filename) as f:
new_people.append(load_yaml(f))
click.secho(
f'analyzing {len(existing_people)} existing people and {len(new_people)} incoming'
)
directory_merge(abbr, existing_people, new_people, remove_identical,
copy_new, interactive)
if old and new:
with open(old) as f:
old_obj = load_yaml(f)
with open(new) as f:
new_obj = load_yaml(f)
if keep not in ('old', 'new'):
raise ValueError('--keep parameter must be old or new')
keep_both_ids = True
if 'incoming' in new:
keep_both_ids = False
merged = merge_people(old_obj,
new_obj,
keep_on_conflict=keep,
keep_both_ids=keep_both_ids)
dump_obj(merged, filename=old)
os.remove(new)
click.secho(
f'merged files into {old}\ndeleted {new}\ncheck git diff before committing'
)
def entrypoint(incoming, old, new, retirement):
"""
Script to assist with merging legislator files.
Can be used in two modes: incoming or file merge.
Incoming mode analyzes incoming/ directory files (generated with to_yaml.py)
and discovers identical & similar files to assist with merging.
File merge mode merges two legislator files.
"""
if incoming:
abbr = incoming
existing_people = []
for filename in glob.glob(
os.path.join(get_data_dir(abbr), "people/*.yml")) + glob.glob(
os.path.join(get_data_dir(abbr), "retired/*.yml")):
with open(filename) as f:
existing_people.append(load_yaml(f))
new_people = []
incoming_dir = get_data_dir(abbr).replace("data", "incoming")
for filename in glob.glob(os.path.join(incoming_dir, "people/*.yml")):
with open(filename) as f:
new_people.append(load_yaml(f))
click.secho(
f"analyzing {len(existing_people)} existing people and {len(new_people)} incoming"
)
unmatched = incoming_merge(abbr, existing_people, new_people,
retirement)
click.secho(f"{len(unmatched)} were unmatched")
if old and new:
with open(old) as f:
old_obj = load_yaml(f)
with open(new) as f:
new_obj = load_yaml(f)
keep_both_ids = True
if "incoming" in new:
keep_both_ids = False
merged = merge_people(old_obj, new_obj, keep_both_ids=keep_both_ids)
dump_obj(merged, filename=old)
os.remove(new)
click.secho(
f"merged files into {old}\ndeleted {new}\ncheck git diff before committing"
)
def to_yaml(input_dir):
"""
Convert scraped JSON in INPUT_DIR to YAML files for this repo.
Will put data into incoming/ directory for usage with merge.py's --incoming option.
"""
# abbr is last piece of directory name
abbr = None
for piece in input_dir.split("/")[::-1]:
if piece:
abbr = piece
break
output_dir = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
output_dir = output_dir.replace("data", "incoming")
assert "incoming" in output_dir
try:
os.makedirs(os.path.join(output_dir, "legislature"))
except FileExistsError:
for file in glob.glob(os.path.join(output_dir, "legislature", "*.yml")):
os.remove(file)
process_dir(input_dir, output_dir, jurisdiction_id)
def to_yaml(input_dir):
"""
Convert pupa scraped JSON in INPUT_DIR to YAML files for this repo.
Will put data into incoming/ directory for usage with merge.py's --incoming option.
"""
# abbr is last piece of directory name
abbr = None
for piece in input_dir.split('/')[::-1]:
if piece:
abbr = piece
break
output_dir = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
output_dir = output_dir.replace('data', 'incoming')
assert 'incoming' in output_dir
for dir in ('people', 'organizations'):
try:
os.makedirs(os.path.join(output_dir, dir))
except FileExistsError:
for file in glob.glob(os.path.join(output_dir, dir, '*.yml')):
os.remove(file)
process_dir(input_dir, output_dir, jurisdiction_id)
def __init__(self, name, node_id):
self.node_id = str(node_id)
self.dir = utils.get_data_dir(name)
if utils.graph_directness[name]:
self.graph = nx.DiGraph()
else:
self.graph = nx.Graph()
def to_csv(abbreviations, upload):
"""
Sync YAML files to DB.
"""
if not abbreviations:
abbreviations = get_all_abbreviations()
if upload:
s3 = boto3.client("s3")
for abbr in abbreviations:
click.secho("==== {} ====".format(abbr), bold=True)
directory = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
person_files = sorted(
glob.glob(os.path.join(directory, "legislature/*.yml")))
fname = f"{abbr}.csv"
write_csv(person_files, jurisdiction_id, fname)
if upload:
s3.upload_file(
fname,
"data.openstates.org",
f"people/current/{abbr}.csv",
ExtraArgs={
"ContentType": "text/csv",
"ACL": "public-read"
},
)
click.secho(
f"uploaded to data.openstates.org/people/current/{abbr}.csv",
fg="green")
def generate_containers_yaml(confDir):
hosts = readSSH()
del hosts["monitoring"]
for host in hosts.keys():
toml = path.join(confDir, host, "katzenpost.toml")
name = host
containerPath = host
image = "hashcloak/meson:master"
if host == "auth":
name = "auth"
containerPath = "nonvoting"
toml = path.join(confDir, "nonvoting", "authority.toml")
image = "hashcloak/authority:master"
ports = [
"{0}:{0}".format(get_mixnet_port(toml)),
"{0}:{0}".format("6543"),
]
if get_user_registration_port(toml):
ports.append("{0}:{0}".format(get_user_registration_port(toml)))
write_service(
containerPath,
generate_service(
image=image,
name=host,
ports=ports,
volumes=["{}:{}".format("/root/config", get_data_dir(toml))]))
def match_ids(abbreviations):
if not abbreviations:
abbreviations = get_all_abbreviations()
for abbr in abbreviations:
click.secho("==== {} ====".format(abbr), bold=True)
m = Matcher(abbr)
for fname in glob.glob(os.path.join(get_data_dir(abbr), "people/*.yml")):
with open(fname) as f:
person = load_yaml(f)
already_done = False
for oid in person.get("other_identifiers", []):
if oid["scheme"] == "legacy_openstates":
already_done = True
break
if already_done:
continue
exact = m.match(person)
if exact:
if "other_identifiers" not in person:
person["other_identifiers"] = []
for id in exact:
person["other_identifiers"].append(
{"scheme": "legacy_openstates", "identifier": id}
)
dump_obj(person, filename=fname)
def remove_retired_data(state):
for filename in glob.glob(
os.path.join(get_data_dir(state), "retired/*.yml")):
with open(filename) as file:
data = load_yaml(file)
data.pop("contact_details", None)
dump_obj(data, filename=filename)
def train(self, config):
print("\nPrepare Data...\n")
input_setup(config)
data_dir = get_data_dir(config.checkpoint_dir, config.is_train)
data_num = get_data_num(data_dir)
images_shape = [None, self.image_size, self.image_size, self.c_dim]
labels_shape = [None, self.image_size * self.scale, self.image_size * self.scale, self.c_dim]
self.build_model(images_shape, labels_shape)
self.train_op = tf.train.AdamOptimizer(learning_rate=config.learning_rate).minimize(self.loss)
tf.global_variables_initializer().run(session=self.sess)
# merged_summary_op = tf.summary.merge_all()
# summary_writer = tf.summary.FileWriter(config.checkpoint_dir, self.sess.graph)
counter = self.load(config.checkpoint_dir)
time_ = time.time()
print("\nNow Start Training...\n")
for ep in range(config.epoch):
# Run by batch images
batch_idxs = data_num // config.batch_size
for idx in range(0, batch_idxs):
batch_images, batch_labels = get_batch(data_dir, data_num, config.batch_size)
counter += 1
_, err = self.sess.run([self.train_op, self.loss], feed_dict={self.images: batch_images, self.labels: batch_labels})
if counter % 10 == 0:
print("Epoch: [%2d], batch: [%2d/%2d], step: [%2d], time: [%4.4f], loss: [%.8f]" % ((ep+1), idx, batch_idxs, counter, time.time()-time_, err))
if counter % 100 == 0:
self.save(config.checkpoint_dir, counter)
# summary_str = self.sess.run(merged_summary_op)
# summary_writer.add_summary(summary_str, counter)
if counter > 0 and counter == batch_idxs * config.epoch:
return
def open_file(window, path=None):
if path==None:
wildcard="|".join([
"All files (*)", "*",
"Hermes2D mesh files (*.mesh)", "*.mesh",
"Agros2D problem files (*.a2d)", "*.a2d",
])
dialog = FileDialog(parent=None, title='Open supported data file',
action='open', wildcard=wildcard,
# use this to have Hermes2D by default:
#wildcard_index=1,
wildcard_index=0,
default_directory=get_data_dir(),
#default_filename="lshape.mesh",
)
if dialog.open() == OK:
path = dialog.path
else:
return
ext = os.path.splitext(path)[1]
if ext == ".a2d":
scene = window.get_view_by_id("Problem")
p, g = read_a2d(path)
scene.problem = p
scene.geometry = g
else:
scene = window.get_view_by_id("Scene")
mesh = read_mesh(path)
scene.mesh = mesh
scene.mode = "mesh"
def test_run():
config = Config(
cuda=False,
data_dir=get_data_dir("gettysburg"),
save_dir=None,
rnn_size=2,
rnn_model="LSTM",
num_layers=2,
batch_size=8,
seq_length=4,
num_epochs=1,
save_every=None,
grad_clip=5.,
learning_rate=0.002,
decay_rate=0.97,
keep_prob=1.0,
sampling_mode="weighted",
)
dataset, model, losses = train_model(config, verbose=False)
hash_result = np.sum(
sample_model(
config=config,
dataset=dataset,
model=model,
text=False,
))
assert hash_result == 18989, hash_result
def to_database(abbreviations, purge, safe):
"""
Sync YAML files to DB.
"""
init_django()
if not abbreviations:
abbreviations = get_all_abbreviations()
settings = get_settings()
for abbr in abbreviations:
click.secho('==== {} ===='.format(abbr), bold=True)
directory = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
person_files = (glob.glob(os.path.join(directory, 'people/*.yml')) +
glob.glob(os.path.join(directory, 'retired/*.yml')))
committee_files = glob.glob(os.path.join(directory, 'organizations/*.yml'))
if safe:
click.secho('running in safe mode, no changes will be made', fg='magenta')
state_settings = settings[abbr]
try:
with transaction.atomic():
create_posts(jurisdiction_id, state_settings)
load_directory(person_files, 'person', jurisdiction_id, purge=purge)
load_directory(committee_files, 'organization', jurisdiction_id, purge=purge)
if safe:
click.secho('ran in safe mode, no changes were made', fg='magenta')
raise CancelTransaction()
except CancelTransaction:
pass
def match_ids(abbreviations):
if not abbreviations:
abbreviations = get_all_abbreviations()
for abbr in abbreviations:
click.secho('==== {} ===='.format(abbr), bold=True)
m = Matcher(abbr)
for fname in glob.glob(os.path.join(get_data_dir(abbr),
'people/*.yml')):
with open(fname) as f:
person = load_yaml(f)
already_done = False
for oid in person.get('other_identifiers', []):
if oid['scheme'] == 'legacy_openstates':
already_done = True
break
if already_done:
continue
exact = m.match(person)
if exact:
if 'other_identifiers' not in person:
person['other_identifiers'] = []
for id in exact:
person['other_identifiers'].append({
'scheme': 'legacy_openstates',
'identifier': id
})
dump_obj(person, filename=fname)
def create_person(fname, lname, name, state, district, party, rtype, url,
image, start_date):
person = OrderedDict({
"id":
ocd_uuid("person"),
"name":
name or f"{fname} {lname}",
"given_name":
fname,
"family_name":
lname,
"image":
image,
"party": [{
"name": party
}],
"roles": [{
"type": rtype,
"district": district,
"jurisdiction": get_jurisdiction_id(state),
"start_date": start_date,
}],
"links": [{
"url": url
}],
"sources": [{
"url": url
}],
})
output_dir = get_data_dir(state)
dump_obj(person, output_dir=os.path.join(output_dir, "people"))
def merge(state, merger):
"""
Merge incoming data for a given state into existing files.
Matches existing people to new people by name. If names match, update
the existing person. For unmatched people, retire existing persons and
create new persons.
"""
data_dir = get_data_dir(state)
existing_people = PersonFile.from_dir(os.path.join(data_dir, 'people')) + \
PersonFile.from_dir(os.path.join(data_dir, 'retired'))
incoming_dir = data_dir.replace('data', 'incoming')
assert data_dir != incoming_dir
new_people = PersonFile.from_dir(os.path.join(incoming_dir, 'people'))
handled = set()
similar = []
for existing in existing_people:
if existing.id in handled:
continue
for new in new_people:
if new.id in handled:
continue
if existing.name == new.name:
if existing.differences(new, new_only=True):
merger.update(existing, new)
handled |= {existing.id, new.id}
break
elif existing.seat == new.seat: # check for similar name, same seat
name_similarity = similarity(existing.name, new.name)
if name_similarity > SIMILAR_NAME_RATIO:
similar.append((name_similarity, existing, new))
similar.sort(key=itemgetter(0), reverse=True)
for _, existing, new in similar:
if existing.id in handled or new.id in handled:
continue
if existing.differences(new, new_only=True):
merger.update(existing, new)
handled |= {existing.id, new.id}
for existing in existing_people:
if existing.id in handled:
continue
if not existing.retired:
merger.retire(existing)
for new in new_people:
if new.id in handled:
continue
merger.create(new)
merger.execute_deferred()
def analyze_lecture():
"""analyzes slides in lecture
Returns:
slide_list: a list of slide objects
"""
delimiter = utils.get_delimiter()
slide_directory = utils.get_screenshot_dir() + delimiter + "teacher"
list_of_files = sorted(os.listdir(slide_directory))
slide_list = []
for i in range(len(list_of_files)):
# print(slide_directory + delimiter + list_of_files[i])
img = cv2.imread(slide_directory + delimiter + list_of_files[i])
# utils.show_image(img)
slide = img_funcs.find_slide(img)
global debug
if debug:
global sharp
global delimite
global debug_dir
cv2.imwrite(
debug_dir + delimiter + 'slide_found-' + str(sharp) + '.jpg',
slide)
sharp += 1
# initialize slide
if i == 0:
slide_list.append(Slide(slide, i + 1))
previous_slide = slide
# check if slide in next frame is the same
else:
if check_if_same_slide(previous_slide, slide):
slide_list.append(slide_list[i - 1])
else:
slide_list.append(Slide(slide, slide_list[i - 1].name + 1))
previous_slide = slide
word_count_and_name = [[int(x.word_count), int(x.name)]
for x in slide_list]
total_slides = len(set([x.name for x in slide_list]))
delimiter = utils.get_delimiter()
data_directory = utils.get_data_dir()
with open(data_directory + delimiter + 'total_slides.txt', 'w') as f:
f.write(str(total_slides))
np.savetxt(data_directory + delimiter + 'slide.csv',
word_count_and_name,
delimiter=',',
fmt='%d',
header='word_count,name')
return slide_list
def train(self, config):
print("\nPrepare Data...\n")
input_setup(config)
data_dir = get_data_dir(config)
data_num = get_data_num(data_dir)
self.train_op = tf.train.AdamOptimizer(
learning_rate=config.learning_rate).minimize(self.loss)
tf.initialize_all_variables().run()
# merged_summary_op = tf.summary.merge_all()
# summary_writer = tf.summary.FileWriter(config.checkpoint_dir, self.sess.graph)
counter = 0
time_ = time.time()
self.load(config.checkpoint_dir)
# Train
if config.is_train:
print("\nNow Start Training...\n")
for ep in range(config.epoch):
# Run by batch images
batch_idxs = data_num // config.batch_size
for idx in range(0, batch_idxs):
batch_images, batch_labels = get_batch(
data_dir, idx, config.batch_size)
counter += 1
_, err = self.sess.run([self.train_op, self.loss],
feed_dict={
self.images: batch_images,
self.labels: batch_labels
})
if counter % 10 == 0:
print(
"Epoch: [%2d], step: [%2d], time: [%4.4f], loss: [%.8f]"
% ((ep + 1), counter, time.time() - time_, err))
if counter % 100 == 0:
self.save(config.checkpoint_dir, counter)
# summary_str = self.sess.run(merged_summary_op)
# summary_writer.add_summary(summary_str, counter)
# Test
else:
print("\nNow Start Testing...\n")
time_ = time.time()
input_, label_ = get_batch(data_dir, 0, 1)
result = self.sess.run([self.pred],
feed_dict={
self.images:
input_[0].reshape(
1, self.h, self.w, self.c_dim)
})
print "time:", (time.time() - time_)
x = np.squeeze(result)
checkimage(x)
print "shape:", x.shape
imsave(x, config.result_dir + '/result.png', config)
def autoretire(abbr):
legislative_filenames = glob.glob(
os.path.join(get_data_dir(abbr), "legislature", "*.yml"))
executive_filenames = glob.glob(
os.path.join(get_data_dir(abbr), "executive", "*.yml"))
municipality_filenames = glob.glob(
os.path.join(get_data_dir(abbr), "municipalities", "*.yml"))
filenames = legislative_filenames + executive_filenames + municipality_filenames
for filename in filenames:
with open(filename) as f:
person = load_yaml(f)
if is_inactive(person):
print("retiring ", filename)
# end_date won't be used since they're already expired
retire_person(person, None)
move_file(filename)
def student_attentiveness():
"""gets the student attentiveness for the lecture
Returns:
a csv of attentiveness
"""
delimiter = utils.get_delimiter()
data_directory = utils.get_data_dir()
screenshot_directory = utils.get_screenshot_dir()
student_directory = screenshot_directory + "/students"
list_of_files = sorted(os.listdir(student_directory))
for i in range(len(list_of_files)):
img = cv2.imread(student_directory + delimiter + list_of_files[i])
if i == 0:
student_list = student_img.initial_frame(img)
for student in student_list:
try:
next_frame = cv2.imread(student_directory + delimiter +
list_of_files[i + 1])
# TODO: Fix redunancy here, find_student_next_frame and find_new_students essentially do the same thing
# without the crop and calling find faces again its way faster
student_img.find_student_next_frame(student, next_frame)
except IndexError:
break
student_img.find_new_students(student_list, next_frame, i + 1)
student_funcs.check_for_absent(student_list)
student_list = student_funcs.check_for_duplicate(student_list)
global debug
if debug:
print(student_directory + delimiter + list_of_files[i])
print(len(student_list))
# removes end of lecture noise
student_list = [
student for student in student_list
if student.present_in_frame > len(list_of_files) * .50
]
# print(len(student_list))
classroom_angles = []
for student in student_list:
student_funcs.get_mode_angle(student)
student_funcs.get_attention_per_frame(student)
classroom_angles.append(student.attention_angle_per_frame)
avg_across_lecture = np.mean(classroom_angles, axis=0)
np.savetxt(data_directory + delimiter + 'attentiveness.csv',
avg_across_lecture,
delimiter=',',
header='attentiveness')
return student_list
def merge_scraped_coms(abbr, old, new):
old_by_key = {(c["parent"], c["name"]): c for c in old}
for c in new:
old_com = old_by_key.pop((c["parent"], c["name"]), None)
if old_com:
old_com["sources"] = c["sources"]
old_com["memberships"] = c["memberships"]
fname = os.path.join(get_data_dir(abbr), "organizations",
get_filename(old_com))
dump_obj(old_com, filename=fname)
click.secho(f"updated {fname}")
os.remove(f"incoming/{abbr}/organizations/{get_filename(c)}")
else:
copy_new_incoming(abbr, c, "organizations")
# remove unmatched old committees
for com in old_by_key.values():
fn = get_filename(com)
click.secho(f"removing {fn}", fg="yellow")
os.remove(os.path.join(get_data_dir(abbr), "organizations", fn))
def process_legislature(self, abbr): # pragma: no cover
filenames = glob.glob(os.path.join(get_data_dir(abbr), "legislature", "*.yml"))
# settings_file = os.path.join(os.path.dirname(__file__), "../settings.yml")
# with open(settings_file) as f:
# settings = load_yaml(f)
for filename in filenames:
with open(filename) as f:
person = load_yaml(f)
self.summarize(person)
def to_csv(abbreviations):
"""
Sync YAML files to DB.
"""
if not abbreviations:
abbreviations = get_all_abbreviations()
for abbr in abbreviations:
click.secho('==== {} ===='.format(abbr), bold=True)
directory = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
person_files = sorted(glob.glob(os.path.join(directory, 'people/*.yml')))
write_csv(person_files, jurisdiction_id, f"csv/{abbr}_legislators.csv")
def to_database(abbreviations, purge, safe):
"""
Sync YAML files to DB.
"""
init_django()
create_parties()
if not abbreviations:
abbreviations = get_all_abbreviations()
for abbr in abbreviations:
click.secho("==== {} ====".format(abbr), bold=True)
directory = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
municipalities = load_municipalities(abbr)
with transaction.atomic():
create_municipalities(municipalities)
person_files = (
glob.glob(os.path.join(directory, "legislature/*.yml")) +
glob.glob(os.path.join(directory, "executive/*.yml")) +
glob.glob(os.path.join(directory, "municipalities/*.yml")) +
glob.glob(os.path.join(directory, "retired/*.yml")))
committee_files = glob.glob(
os.path.join(directory, "organizations/*.yml"))
if safe:
click.secho("running in safe mode, no changes will be made",
fg="magenta")
try:
with transaction.atomic():
load_directory(person_files,
"person",
jurisdiction_id,
purge=purge)
load_directory(committee_files,
"organization",
jurisdiction_id,
purge=purge)
if safe:
click.secho("ran in safe mode, no changes were made",
fg="magenta")
raise CancelTransaction()
except CancelTransaction:
sys.exit(1)
def load_person_by_id(abbr, person_id):
directory = get_data_dir(abbr)
person_id = person_id.replace("ocd-person/", "")
person = glob.glob(os.path.join(directory, "*", f"*{person_id}.yml"))
if len(person) < 1:
click.secho(f"could not find {abbr} {person_id}")
return None, None
elif len(person) > 1:
click.secho(f"multiple matches for {abbr} {person_id}")
return None, None
# found them, load & return
with open(person[0]) as f:
return person[0], load_yaml(f)
def to_database(abbr, verbose, summary, purge, safe):
init_django()
directory = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
person_files = (glob.glob(os.path.join(directory, 'people/*.yml')) +
glob.glob(os.path.join(directory, 'retired/*.yml')))
committee_files = glob.glob(os.path.join(directory, 'organizations/*.yml'))
if safe:
click.secho('running in safe mode, no changes will be made', fg='magenta')
try:
with transaction.atomic():
load_directory(person_files, 'person', jurisdiction_id, purge=purge)
load_directory(committee_files, 'organization', jurisdiction_id, purge=purge)
if safe:
click.secho('ran in safe mode, no changes were made', fg='magenta')
raise CancelTransaction()
except CancelTransaction:
pass
def to_yaml(input_dir, reset):
# TODO: remove reset option once we're in prod
# abbr is last piece of directory name
abbr = None
for piece in input_dir.split('/')[::-1]:
if piece:
abbr = piece
break
output_dir = get_data_dir(abbr)
jurisdiction_id = get_jurisdiction_id(abbr)
for dir in ('people', 'organizations'):
try:
os.makedirs(os.path.join(output_dir, dir))
except FileExistsError:
if reset:
for file in glob.glob(os.path.join(output_dir, dir, '*.yml')):
os.remove(file)
process_dir(input_dir, output_dir, jurisdiction_id)
def __init(self, *args):
self.debug = args[0] or os.getenv("GGET_DEBUG")
self.base_dir = os.path.expanduser(os.path.join(XDG_CONFIG_DIR,
"gget"))
if not os.path.exists(self.base_dir):
os.makedirs(self.base_dir)
self.client = gconf.client_get_default()
if self.client.dir_exists(DIR_GGET):
self.client.add_dir(DIR_GGET, gconf.CLIENT_PRELOAD_RECURSIVE)
else:
if utils.runned_from_source():
os.system("gconftool-2 --install-schema-file %s" %
os.path.join(utils.get_data_dir(), "gget.schemas"))
else:
ed = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, buttons=gtk.BUTTONS_OK, message_format=_("Could not find configuration directory in GConf"))
ed.format_secondary_text(_("Please make sure that gget.schemas was correctly installed."))
ed.set_title(_("Error"))
#ed = dialogs.ErrorDialog(_("Could not find configuration directory in GConf"), _("Please make sure that gget.schemas was correctly installed."))
ed.run()
sys.exit(1)
import os.path
from gettext import gettext as _
import gtk
import gtk.gdk
import gtk.glade
import gnomevfs
import gnome.ui
import utils
from gget import NAME, LOCALE_DIR
gtk.glade.bindtextdomain(NAME.lower(), LOCALE_DIR)
glade_file = os.path.join(utils.get_data_dir(), "gget.glade")
icon_theme = gtk.icon_theme_get_default()
def get_icon_list(sizes):
icon_list = []
for size in sizes:
icon_list.append(load_icon(NAME.lower(), size, size))
return icon_list
def load_icon(icon, width=48, height=48):
pixbuf = None
if icon != None and icon != "":
try:
icon_file = os.path.join(utils.get_images_dir(), icon)
if os.path.exists(icon_file):
def get_dictionary(dict_filename="cpv2010arg_diccionario.json"):
with open(os.path.join(get_data_dir(), dict_filename), "r") as f:
return json.load(f)
def get_ids(ids_filename="cpv2010arg_ids.json"):
with open(os.path.join(get_data_dir(), ids_filename), "r") as f:
return json.load(f)
Item("nodes"),
Item("edges"),
Item("labels"),
resizable=True)
def read_a2d(filename):
root = etree.fromstring(open(filename).read())
problem = root.find("problems").find("problem")
p = Problem(**problem.attrib)
p.edges = [ProblemEdge(**edge.attrib) for edge in problem.find("edges")]
p.labels = [ProblemLabel(**label.attrib) for label in \
problem.find("labels")]
geometry = root.find("geometry")
g = Geometry()
g.nodes = [Node(**node.attrib) for node in geometry.find("nodes")]
g.edges = [Edge(**edge.attrib) for edge in geometry.find("edges")]
g.labels = [Label(**label.attrib) for label in geometry.find("labels")]
return p, g
if __name__ == "__main__":
import glob
for file in glob.glob(get_data_dir()+"/agros2d/*.a2d"):
print file
read_a2d(file)
