def process_dir(abbr, verbose, summary, settings): person_filenames = glob.glob( os.path.join(get_data_dir(abbr), 'people', '*.yml')) retired_filenames = glob.glob( os.path.join(get_data_dir(abbr), 'retired', '*.yml')) org_filenames = glob.glob( os.path.join(get_data_dir(abbr), 'organizations', '*.yml')) validator = Validator(settings, abbr) for filename in person_filenames: print_filename = os.path.basename(filename) with open(filename) as f: person = yaml.load(f) validator.validate_person(person, print_filename) for filename in retired_filenames: print_filename = os.path.basename(filename) with open(filename) as f: person = yaml.load(f) validator.validate_person(person, print_filename, retired=True) for filename in org_filenames: print_filename = os.path.basename(filename) with open(filename) as f: org = yaml.load(f) validator.validate_org(org, print_filename) validator.print_validation_report(verbose) if summary: validator.print_summary()
def process_dir(abbr, verbose, municipal, date): # pragma: no cover legislative_filenames = glob.glob(os.path.join(get_data_dir(abbr), "legislature", "*.yml")) executive_filenames = glob.glob(os.path.join(get_data_dir(abbr), "executive", "*.yml")) municipality_filenames = glob.glob(os.path.join(get_data_dir(abbr), "municipalities", "*.yml")) retired_filenames = glob.glob(os.path.join(get_data_dir(abbr), "retired", "*.yml")) settings_file = os.path.join(os.path.dirname(__file__), "../settings.yml") with open(settings_file) as f: settings = load_yaml(f) try: validator = Validator(abbr, settings) except BadVacancy: sys.exit(-1) all_filenames = [ (PersonType.LEGISLATIVE, legislative_filenames), (PersonType.RETIRED, retired_filenames), (PersonType.EXECUTIVE, executive_filenames), ] if municipal: all_filenames.append((PersonType.MUNICIPAL, municipality_filenames)) for person_type, filenames in all_filenames: for filename in filenames: print_filename = os.path.basename(filename) with open(filename) as f: person = load_yaml(f) validator.validate_person(person, print_filename, person_type, date) error_count = validator.print_validation_report(verbose) return error_count
def process_dir(abbr, verbose, summary, settings): # pragma: no cover person_filenames = glob.glob(os.path.join(get_data_dir(abbr), "people", "*.yml")) retired_filenames = glob.glob(os.path.join(get_data_dir(abbr), "retired", "*.yml")) org_filenames = glob.glob(os.path.join(get_data_dir(abbr), "organizations", "*.yml")) try: validator = Validator(abbr, settings) except BadVacancy: sys.exit(-1) for filename in person_filenames: print_filename = os.path.basename(filename) with open(filename) as f: person = load_yaml(f) validator.validate_person(person, print_filename) for filename in retired_filenames: print_filename = os.path.basename(filename) with open(filename) as f: person = load_yaml(f) validator.validate_person(person, print_filename, retired=True) for filename in org_filenames: print_filename = os.path.basename(filename) with open(filename) as f: org = load_yaml(f) validator.validate_org(org, print_filename) error_count = validator.print_validation_report(verbose) if summary: validator.print_summary() return error_count
def find_by_id(id): id = id.split("/")[1] choices = glob.glob( os.path.join(get_data_dir("*"), "people", f"*-{id}.yml")) + glob.glob( os.path.join(get_data_dir("*"), "retired", f"*-{id}.yml")) if len(choices) != 1: raise ValueError(f"unknown id {id}") return choices[0]
def find_by_id(id): id = id.split('/')[1] choices = glob.glob( os.path.join(get_data_dir('*'), 'people', f'*-{id}.yml')) + glob.glob( os.path.join(get_data_dir("*"), "retired", f"*-{id}.yml")) if len(choices) != 1: raise ValueError(f'unknown id {id}') return choices[0]
def entrypoint(incoming, old, new, keep, remove_identical, copy_new, interactive): """ Script to assist with merging legislator files. Can be used in two modes: incoming or file merge. Incoming mode analyzes incoming/ directory files (generated with to_yaml.py) and discovers identical & similar files to assist with merging. File merge mode merges two legislator files. """ if incoming: abbr = incoming existing_people = [] for filename in ( glob.glob(os.path.join(get_data_dir(abbr), 'people/*.yml')) + glob.glob(os.path.join(get_data_dir(abbr), 'retired/*.yml'))): with open(filename) as f: existing_people.append(load_yaml(f)) new_people = [] incoming_dir = get_data_dir(abbr).replace('data', 'incoming') for filename in glob.glob(os.path.join(incoming_dir, 'people/*.yml')): with open(filename) as f: new_people.append(load_yaml(f)) click.secho( f'analyzing {len(existing_people)} existing people and {len(new_people)} incoming' ) directory_merge(abbr, existing_people, new_people, remove_identical, copy_new, interactive) if old and new: with open(old) as f: old_obj = load_yaml(f) with open(new) as f: new_obj = load_yaml(f) if keep not in ('old', 'new'): raise ValueError('--keep parameter must be old or new') keep_both_ids = True if 'incoming' in new: keep_both_ids = False merged = merge_people(old_obj, new_obj, keep_on_conflict=keep, keep_both_ids=keep_both_ids) dump_obj(merged, filename=old) os.remove(new) click.secho( f'merged files into {old}\ndeleted {new}\ncheck git diff before committing' )
def entrypoint(incoming, old, new, retirement): """ Script to assist with merging legislator files. Can be used in two modes: incoming or file merge. Incoming mode analyzes incoming/ directory files (generated with to_yaml.py) and discovers identical & similar files to assist with merging. File merge mode merges two legislator files. """ if incoming: abbr = incoming existing_people = [] for filename in glob.glob( os.path.join(get_data_dir(abbr), "people/*.yml")) + glob.glob( os.path.join(get_data_dir(abbr), "retired/*.yml")): with open(filename) as f: existing_people.append(load_yaml(f)) new_people = [] incoming_dir = get_data_dir(abbr).replace("data", "incoming") for filename in glob.glob(os.path.join(incoming_dir, "people/*.yml")): with open(filename) as f: new_people.append(load_yaml(f)) click.secho( f"analyzing {len(existing_people)} existing people and {len(new_people)} incoming" ) unmatched = incoming_merge(abbr, existing_people, new_people, retirement) click.secho(f"{len(unmatched)} were unmatched") if old and new: with open(old) as f: old_obj = load_yaml(f) with open(new) as f: new_obj = load_yaml(f) keep_both_ids = True if "incoming" in new: keep_both_ids = False merged = merge_people(old_obj, new_obj, keep_both_ids=keep_both_ids) dump_obj(merged, filename=old) os.remove(new) click.secho( f"merged files into {old}\ndeleted {new}\ncheck git diff before committing" )
def to_yaml(input_dir): """ Convert scraped JSON in INPUT_DIR to YAML files for this repo. Will put data into incoming/ directory for usage with merge.py's --incoming option. """ # abbr is last piece of directory name abbr = None for piece in input_dir.split("/")[::-1]: if piece: abbr = piece break output_dir = get_data_dir(abbr) jurisdiction_id = get_jurisdiction_id(abbr) output_dir = output_dir.replace("data", "incoming") assert "incoming" in output_dir try: os.makedirs(os.path.join(output_dir, "legislature")) except FileExistsError: for file in glob.glob(os.path.join(output_dir, "legislature", "*.yml")): os.remove(file) process_dir(input_dir, output_dir, jurisdiction_id)
def to_yaml(input_dir): """ Convert pupa scraped JSON in INPUT_DIR to YAML files for this repo. Will put data into incoming/ directory for usage with merge.py's --incoming option. """ # abbr is last piece of directory name abbr = None for piece in input_dir.split('/')[::-1]: if piece: abbr = piece break output_dir = get_data_dir(abbr) jurisdiction_id = get_jurisdiction_id(abbr) output_dir = output_dir.replace('data', 'incoming') assert 'incoming' in output_dir for dir in ('people', 'organizations'): try: os.makedirs(os.path.join(output_dir, dir)) except FileExistsError: for file in glob.glob(os.path.join(output_dir, dir, '*.yml')): os.remove(file) process_dir(input_dir, output_dir, jurisdiction_id)
def __init__(self, name, node_id): self.node_id = str(node_id) self.dir = utils.get_data_dir(name) if utils.graph_directness[name]: self.graph = nx.DiGraph() else: self.graph = nx.Graph()
def to_csv(abbreviations, upload): """ Sync YAML files to DB. """ if not abbreviations: abbreviations = get_all_abbreviations() if upload: s3 = boto3.client("s3") for abbr in abbreviations: click.secho("==== {} ====".format(abbr), bold=True) directory = get_data_dir(abbr) jurisdiction_id = get_jurisdiction_id(abbr) person_files = sorted( glob.glob(os.path.join(directory, "legislature/*.yml"))) fname = f"{abbr}.csv" write_csv(person_files, jurisdiction_id, fname) if upload: s3.upload_file( fname, "data.openstates.org", f"people/current/{abbr}.csv", ExtraArgs={ "ContentType": "text/csv", "ACL": "public-read" }, ) click.secho( f"uploaded to data.openstates.org/people/current/{abbr}.csv", fg="green")
def generate_containers_yaml(confDir): hosts = readSSH() del hosts["monitoring"] for host in hosts.keys(): toml = path.join(confDir, host, "katzenpost.toml") name = host containerPath = host image = "hashcloak/meson:master" if host == "auth": name = "auth" containerPath = "nonvoting" toml = path.join(confDir, "nonvoting", "authority.toml") image = "hashcloak/authority:master" ports = [ "{0}:{0}".format(get_mixnet_port(toml)), "{0}:{0}".format("6543"), ] if get_user_registration_port(toml): ports.append("{0}:{0}".format(get_user_registration_port(toml))) write_service( containerPath, generate_service( image=image, name=host, ports=ports, volumes=["{}:{}".format("/root/config", get_data_dir(toml))]))
def match_ids(abbreviations): if not abbreviations: abbreviations = get_all_abbreviations() for abbr in abbreviations: click.secho("==== {} ====".format(abbr), bold=True) m = Matcher(abbr) for fname in glob.glob(os.path.join(get_data_dir(abbr), "people/*.yml")): with open(fname) as f: person = load_yaml(f) already_done = False for oid in person.get("other_identifiers", []): if oid["scheme"] == "legacy_openstates": already_done = True break if already_done: continue exact = m.match(person) if exact: if "other_identifiers" not in person: person["other_identifiers"] = [] for id in exact: person["other_identifiers"].append( {"scheme": "legacy_openstates", "identifier": id} ) dump_obj(person, filename=fname)
def remove_retired_data(state): for filename in glob.glob( os.path.join(get_data_dir(state), "retired/*.yml")): with open(filename) as file: data = load_yaml(file) data.pop("contact_details", None) dump_obj(data, filename=filename)
def train(self, config): print("\nPrepare Data...\n") input_setup(config) data_dir = get_data_dir(config.checkpoint_dir, config.is_train) data_num = get_data_num(data_dir) images_shape = [None, self.image_size, self.image_size, self.c_dim] labels_shape = [None, self.image_size * self.scale, self.image_size * self.scale, self.c_dim] self.build_model(images_shape, labels_shape) self.train_op = tf.train.AdamOptimizer(learning_rate=config.learning_rate).minimize(self.loss) tf.global_variables_initializer().run(session=self.sess) # merged_summary_op = tf.summary.merge_all() # summary_writer = tf.summary.FileWriter(config.checkpoint_dir, self.sess.graph) counter = self.load(config.checkpoint_dir) time_ = time.time() print("\nNow Start Training...\n") for ep in range(config.epoch): # Run by batch images batch_idxs = data_num // config.batch_size for idx in range(0, batch_idxs): batch_images, batch_labels = get_batch(data_dir, data_num, config.batch_size) counter += 1 _, err = self.sess.run([self.train_op, self.loss], feed_dict={self.images: batch_images, self.labels: batch_labels}) if counter % 10 == 0: print("Epoch: [%2d], batch: [%2d/%2d], step: [%2d], time: [%4.4f], loss: [%.8f]" % ((ep+1), idx, batch_idxs, counter, time.time()-time_, err)) if counter % 100 == 0: self.save(config.checkpoint_dir, counter) # summary_str = self.sess.run(merged_summary_op) # summary_writer.add_summary(summary_str, counter) if counter > 0 and counter == batch_idxs * config.epoch: return
def open_file(window, path=None): if path==None: wildcard="|".join([ "All files (*)", "*", "Hermes2D mesh files (*.mesh)", "*.mesh", "Agros2D problem files (*.a2d)", "*.a2d", ]) dialog = FileDialog(parent=None, title='Open supported data file', action='open', wildcard=wildcard, # use this to have Hermes2D by default: #wildcard_index=1, wildcard_index=0, default_directory=get_data_dir(), #default_filename="lshape.mesh", ) if dialog.open() == OK: path = dialog.path else: return ext = os.path.splitext(path)[1] if ext == ".a2d": scene = window.get_view_by_id("Problem") p, g = read_a2d(path) scene.problem = p scene.geometry = g else: scene = window.get_view_by_id("Scene") mesh = read_mesh(path) scene.mesh = mesh scene.mode = "mesh"
def test_run(): config = Config( cuda=False, data_dir=get_data_dir("gettysburg"), save_dir=None, rnn_size=2, rnn_model="LSTM", num_layers=2, batch_size=8, seq_length=4, num_epochs=1, save_every=None, grad_clip=5., learning_rate=0.002, decay_rate=0.97, keep_prob=1.0, sampling_mode="weighted", ) dataset, model, losses = train_model(config, verbose=False) hash_result = np.sum( sample_model( config=config, dataset=dataset, model=model, text=False, )) assert hash_result == 18989, hash_result
def to_database(abbreviations, purge, safe): """ Sync YAML files to DB. """ init_django() if not abbreviations: abbreviations = get_all_abbreviations() settings = get_settings() for abbr in abbreviations: click.secho('==== {} ===='.format(abbr), bold=True) directory = get_data_dir(abbr) jurisdiction_id = get_jurisdiction_id(abbr) person_files = (glob.glob(os.path.join(directory, 'people/*.yml')) + glob.glob(os.path.join(directory, 'retired/*.yml'))) committee_files = glob.glob(os.path.join(directory, 'organizations/*.yml')) if safe: click.secho('running in safe mode, no changes will be made', fg='magenta') state_settings = settings[abbr] try: with transaction.atomic(): create_posts(jurisdiction_id, state_settings) load_directory(person_files, 'person', jurisdiction_id, purge=purge) load_directory(committee_files, 'organization', jurisdiction_id, purge=purge) if safe: click.secho('ran in safe mode, no changes were made', fg='magenta') raise CancelTransaction() except CancelTransaction: pass
def match_ids(abbreviations): if not abbreviations: abbreviations = get_all_abbreviations() for abbr in abbreviations: click.secho('==== {} ===='.format(abbr), bold=True) m = Matcher(abbr) for fname in glob.glob(os.path.join(get_data_dir(abbr), 'people/*.yml')): with open(fname) as f: person = load_yaml(f) already_done = False for oid in person.get('other_identifiers', []): if oid['scheme'] == 'legacy_openstates': already_done = True break if already_done: continue exact = m.match(person) if exact: if 'other_identifiers' not in person: person['other_identifiers'] = [] for id in exact: person['other_identifiers'].append({ 'scheme': 'legacy_openstates', 'identifier': id }) dump_obj(person, filename=fname)
def create_person(fname, lname, name, state, district, party, rtype, url, image, start_date): person = OrderedDict({ "id": ocd_uuid("person"), "name": name or f"{fname} {lname}", "given_name": fname, "family_name": lname, "image": image, "party": [{ "name": party }], "roles": [{ "type": rtype, "district": district, "jurisdiction": get_jurisdiction_id(state), "start_date": start_date, }], "links": [{ "url": url }], "sources": [{ "url": url }], }) output_dir = get_data_dir(state) dump_obj(person, output_dir=os.path.join(output_dir, "people"))
def merge(state, merger): """ Merge incoming data for a given state into existing files. Matches existing people to new people by name. If names match, update the existing person. For unmatched people, retire existing persons and create new persons. """ data_dir = get_data_dir(state) existing_people = PersonFile.from_dir(os.path.join(data_dir, 'people')) + \ PersonFile.from_dir(os.path.join(data_dir, 'retired')) incoming_dir = data_dir.replace('data', 'incoming') assert data_dir != incoming_dir new_people = PersonFile.from_dir(os.path.join(incoming_dir, 'people')) handled = set() similar = [] for existing in existing_people: if existing.id in handled: continue for new in new_people: if new.id in handled: continue if existing.name == new.name: if existing.differences(new, new_only=True): merger.update(existing, new) handled |= {existing.id, new.id} break elif existing.seat == new.seat: # check for similar name, same seat name_similarity = similarity(existing.name, new.name) if name_similarity > SIMILAR_NAME_RATIO: similar.append((name_similarity, existing, new)) similar.sort(key=itemgetter(0), reverse=True) for _, existing, new in similar: if existing.id in handled or new.id in handled: continue if existing.differences(new, new_only=True): merger.update(existing, new) handled |= {existing.id, new.id} for existing in existing_people: if existing.id in handled: continue if not existing.retired: merger.retire(existing) for new in new_people: if new.id in handled: continue merger.create(new) merger.execute_deferred()
def analyze_lecture(): """analyzes slides in lecture Returns: slide_list: a list of slide objects """ delimiter = utils.get_delimiter() slide_directory = utils.get_screenshot_dir() + delimiter + "teacher" list_of_files = sorted(os.listdir(slide_directory)) slide_list = [] for i in range(len(list_of_files)): # print(slide_directory + delimiter + list_of_files[i]) img = cv2.imread(slide_directory + delimiter + list_of_files[i]) # utils.show_image(img) slide = img_funcs.find_slide(img) global debug if debug: global sharp global delimite global debug_dir cv2.imwrite( debug_dir + delimiter + 'slide_found-' + str(sharp) + '.jpg', slide) sharp += 1 # initialize slide if i == 0: slide_list.append(Slide(slide, i + 1)) previous_slide = slide # check if slide in next frame is the same else: if check_if_same_slide(previous_slide, slide): slide_list.append(slide_list[i - 1]) else: slide_list.append(Slide(slide, slide_list[i - 1].name + 1)) previous_slide = slide word_count_and_name = [[int(x.word_count), int(x.name)] for x in slide_list] total_slides = len(set([x.name for x in slide_list])) delimiter = utils.get_delimiter() data_directory = utils.get_data_dir() with open(data_directory + delimiter + 'total_slides.txt', 'w') as f: f.write(str(total_slides)) np.savetxt(data_directory + delimiter + 'slide.csv', word_count_and_name, delimiter=',', fmt='%d', header='word_count,name') return slide_list
def train(self, config): print("\nPrepare Data...\n") input_setup(config) data_dir = get_data_dir(config) data_num = get_data_num(data_dir) self.train_op = tf.train.AdamOptimizer( learning_rate=config.learning_rate).minimize(self.loss) tf.initialize_all_variables().run() # merged_summary_op = tf.summary.merge_all() # summary_writer = tf.summary.FileWriter(config.checkpoint_dir, self.sess.graph) counter = 0 time_ = time.time() self.load(config.checkpoint_dir) # Train if config.is_train: print("\nNow Start Training...\n") for ep in range(config.epoch): # Run by batch images batch_idxs = data_num // config.batch_size for idx in range(0, batch_idxs): batch_images, batch_labels = get_batch( data_dir, idx, config.batch_size) counter += 1 _, err = self.sess.run([self.train_op, self.loss], feed_dict={ self.images: batch_images, self.labels: batch_labels }) if counter % 10 == 0: print( "Epoch: [%2d], step: [%2d], time: [%4.4f], loss: [%.8f]" % ((ep + 1), counter, time.time() - time_, err)) if counter % 100 == 0: self.save(config.checkpoint_dir, counter) # summary_str = self.sess.run(merged_summary_op) # summary_writer.add_summary(summary_str, counter) # Test else: print("\nNow Start Testing...\n") time_ = time.time() input_, label_ = get_batch(data_dir, 0, 1) result = self.sess.run([self.pred], feed_dict={ self.images: input_[0].reshape( 1, self.h, self.w, self.c_dim) }) print "time:", (time.time() - time_) x = np.squeeze(result) checkimage(x) print "shape:", x.shape imsave(x, config.result_dir + '/result.png', config)
def autoretire(abbr): legislative_filenames = glob.glob( os.path.join(get_data_dir(abbr), "legislature", "*.yml")) executive_filenames = glob.glob( os.path.join(get_data_dir(abbr), "executive", "*.yml")) municipality_filenames = glob.glob( os.path.join(get_data_dir(abbr), "municipalities", "*.yml")) filenames = legislative_filenames + executive_filenames + municipality_filenames for filename in filenames: with open(filename) as f: person = load_yaml(f) if is_inactive(person): print("retiring ", filename) # end_date won't be used since they're already expired retire_person(person, None) move_file(filename)
def student_attentiveness(): """gets the student attentiveness for the lecture Returns: a csv of attentiveness """ delimiter = utils.get_delimiter() data_directory = utils.get_data_dir() screenshot_directory = utils.get_screenshot_dir() student_directory = screenshot_directory + "/students" list_of_files = sorted(os.listdir(student_directory)) for i in range(len(list_of_files)): img = cv2.imread(student_directory + delimiter + list_of_files[i]) if i == 0: student_list = student_img.initial_frame(img) for student in student_list: try: next_frame = cv2.imread(student_directory + delimiter + list_of_files[i + 1]) # TODO: Fix redunancy here, find_student_next_frame and find_new_students essentially do the same thing # without the crop and calling find faces again its way faster student_img.find_student_next_frame(student, next_frame) except IndexError: break student_img.find_new_students(student_list, next_frame, i + 1) student_funcs.check_for_absent(student_list) student_list = student_funcs.check_for_duplicate(student_list) global debug if debug: print(student_directory + delimiter + list_of_files[i]) print(len(student_list)) # removes end of lecture noise student_list = [ student for student in student_list if student.present_in_frame > len(list_of_files) * .50 ] # print(len(student_list)) classroom_angles = [] for student in student_list: student_funcs.get_mode_angle(student) student_funcs.get_attention_per_frame(student) classroom_angles.append(student.attention_angle_per_frame) avg_across_lecture = np.mean(classroom_angles, axis=0) np.savetxt(data_directory + delimiter + 'attentiveness.csv', avg_across_lecture, delimiter=',', header='attentiveness') return student_list
def merge_scraped_coms(abbr, old, new): old_by_key = {(c["parent"], c["name"]): c for c in old} for c in new: old_com = old_by_key.pop((c["parent"], c["name"]), None) if old_com: old_com["sources"] = c["sources"] old_com["memberships"] = c["memberships"] fname = os.path.join(get_data_dir(abbr), "organizations", get_filename(old_com)) dump_obj(old_com, filename=fname) click.secho(f"updated {fname}") os.remove(f"incoming/{abbr}/organizations/{get_filename(c)}") else: copy_new_incoming(abbr, c, "organizations") # remove unmatched old committees for com in old_by_key.values(): fn = get_filename(com) click.secho(f"removing {fn}", fg="yellow") os.remove(os.path.join(get_data_dir(abbr), "organizations", fn))
def process_legislature(self, abbr): # pragma: no cover filenames = glob.glob(os.path.join(get_data_dir(abbr), "legislature", "*.yml")) # settings_file = os.path.join(os.path.dirname(__file__), "../settings.yml") # with open(settings_file) as f: # settings = load_yaml(f) for filename in filenames: with open(filename) as f: person = load_yaml(f) self.summarize(person)
def to_csv(abbreviations): """ Sync YAML files to DB. """ if not abbreviations: abbreviations = get_all_abbreviations() for abbr in abbreviations: click.secho('==== {} ===='.format(abbr), bold=True) directory = get_data_dir(abbr) jurisdiction_id = get_jurisdiction_id(abbr) person_files = sorted(glob.glob(os.path.join(directory, 'people/*.yml'))) write_csv(person_files, jurisdiction_id, f"csv/{abbr}_legislators.csv")
def to_database(abbreviations, purge, safe): """ Sync YAML files to DB. """ init_django() create_parties() if not abbreviations: abbreviations = get_all_abbreviations() for abbr in abbreviations: click.secho("==== {} ====".format(abbr), bold=True) directory = get_data_dir(abbr) jurisdiction_id = get_jurisdiction_id(abbr) municipalities = load_municipalities(abbr) with transaction.atomic(): create_municipalities(municipalities) person_files = ( glob.glob(os.path.join(directory, "legislature/*.yml")) + glob.glob(os.path.join(directory, "executive/*.yml")) + glob.glob(os.path.join(directory, "municipalities/*.yml")) + glob.glob(os.path.join(directory, "retired/*.yml"))) committee_files = glob.glob( os.path.join(directory, "organizations/*.yml")) if safe: click.secho("running in safe mode, no changes will be made", fg="magenta") try: with transaction.atomic(): load_directory(person_files, "person", jurisdiction_id, purge=purge) load_directory(committee_files, "organization", jurisdiction_id, purge=purge) if safe: click.secho("ran in safe mode, no changes were made", fg="magenta") raise CancelTransaction() except CancelTransaction: sys.exit(1)
def load_person_by_id(abbr, person_id): directory = get_data_dir(abbr) person_id = person_id.replace("ocd-person/", "") person = glob.glob(os.path.join(directory, "*", f"*{person_id}.yml")) if len(person) < 1: click.secho(f"could not find {abbr} {person_id}") return None, None elif len(person) > 1: click.secho(f"multiple matches for {abbr} {person_id}") return None, None # found them, load & return with open(person[0]) as f: return person[0], load_yaml(f)
def to_database(abbr, verbose, summary, purge, safe): init_django() directory = get_data_dir(abbr) jurisdiction_id = get_jurisdiction_id(abbr) person_files = (glob.glob(os.path.join(directory, 'people/*.yml')) + glob.glob(os.path.join(directory, 'retired/*.yml'))) committee_files = glob.glob(os.path.join(directory, 'organizations/*.yml')) if safe: click.secho('running in safe mode, no changes will be made', fg='magenta') try: with transaction.atomic(): load_directory(person_files, 'person', jurisdiction_id, purge=purge) load_directory(committee_files, 'organization', jurisdiction_id, purge=purge) if safe: click.secho('ran in safe mode, no changes were made', fg='magenta') raise CancelTransaction() except CancelTransaction: pass
def to_yaml(input_dir, reset): # TODO: remove reset option once we're in prod # abbr is last piece of directory name abbr = None for piece in input_dir.split('/')[::-1]: if piece: abbr = piece break output_dir = get_data_dir(abbr) jurisdiction_id = get_jurisdiction_id(abbr) for dir in ('people', 'organizations'): try: os.makedirs(os.path.join(output_dir, dir)) except FileExistsError: if reset: for file in glob.glob(os.path.join(output_dir, dir, '*.yml')): os.remove(file) process_dir(input_dir, output_dir, jurisdiction_id)
def __init(self, *args): self.debug = args[0] or os.getenv("GGET_DEBUG") self.base_dir = os.path.expanduser(os.path.join(XDG_CONFIG_DIR, "gget")) if not os.path.exists(self.base_dir): os.makedirs(self.base_dir) self.client = gconf.client_get_default() if self.client.dir_exists(DIR_GGET): self.client.add_dir(DIR_GGET, gconf.CLIENT_PRELOAD_RECURSIVE) else: if utils.runned_from_source(): os.system("gconftool-2 --install-schema-file %s" % os.path.join(utils.get_data_dir(), "gget.schemas")) else: ed = gtk.MessageDialog(type=gtk.MESSAGE_ERROR, buttons=gtk.BUTTONS_OK, message_format=_("Could not find configuration directory in GConf")) ed.format_secondary_text(_("Please make sure that gget.schemas was correctly installed.")) ed.set_title(_("Error")) #ed = dialogs.ErrorDialog(_("Could not find configuration directory in GConf"), _("Please make sure that gget.schemas was correctly installed.")) ed.run() sys.exit(1)
import os.path from gettext import gettext as _ import gtk import gtk.gdk import gtk.glade import gnomevfs import gnome.ui import utils from gget import NAME, LOCALE_DIR gtk.glade.bindtextdomain(NAME.lower(), LOCALE_DIR) glade_file = os.path.join(utils.get_data_dir(), "gget.glade") icon_theme = gtk.icon_theme_get_default() def get_icon_list(sizes): icon_list = [] for size in sizes: icon_list.append(load_icon(NAME.lower(), size, size)) return icon_list def load_icon(icon, width=48, height=48): pixbuf = None if icon != None and icon != "": try: icon_file = os.path.join(utils.get_images_dir(), icon) if os.path.exists(icon_file):
def get_dictionary(dict_filename="cpv2010arg_diccionario.json"): with open(os.path.join(get_data_dir(), dict_filename), "r") as f: return json.load(f)
def get_ids(ids_filename="cpv2010arg_ids.json"): with open(os.path.join(get_data_dir(), ids_filename), "r") as f: return json.load(f)
Item("nodes"), Item("edges"), Item("labels"), resizable=True) def read_a2d(filename): root = etree.fromstring(open(filename).read()) problem = root.find("problems").find("problem") p = Problem(**problem.attrib) p.edges = [ProblemEdge(**edge.attrib) for edge in problem.find("edges")] p.labels = [ProblemLabel(**label.attrib) for label in \ problem.find("labels")] geometry = root.find("geometry") g = Geometry() g.nodes = [Node(**node.attrib) for node in geometry.find("nodes")] g.edges = [Edge(**edge.attrib) for edge in geometry.find("edges")] g.labels = [Label(**label.attrib) for label in geometry.find("labels")] return p, g if __name__ == "__main__": import glob for file in glob.glob(get_data_dir()+"/agros2d/*.a2d"): print file read_a2d(file)