def relay(): """ request.json = [int/float]*310 """ global action state = get_entities(request.json) me, opp, good_tower, bad_tower, stats, creeps, allies, enemies = state d = 2000 current_observation = [ *[abs(me.x - e.x) / d for e in [opp, good_tower, bad_tower] + creeps], *[abs(me.y - e.y) / d for e in [opp, good_tower, bad_tower] + creeps], *[abs(creep.x - bad_tower.x) / d for creep in allies], *[abs(creep.y - bad_tower.y) / d for creep in allies], me.hp / me.maxhp, me.mana / me.abilities[0].cost, opp.hp / opp.maxhp, *[creep.hp / me.dmg for creep in creeps] ] observations, states = list(zip(last4states)) observation = [i for j in observations for i in j] last4states.append((current_observation, state)) if stats.time < 10 or abs(me.xy - good_tower.xy) > d: return {'actionCode': 1} legal_actions = get_legal_moves(me, opp, allies, enemies, bad_tower, stats) observations, states = list(zip(last4states)) observation_ = [i for j in observations for i in j] r = reward(*states[-2:]) agent.memory.append((observation, action, r, obeservation_)) action = agent.predict(last4states, legal_actions)} return {'actionCode': action}
def prepare_and_launch(file_path): # get access token and input to headers for requests headers = {"Authorization": "bearer " + get_access_token()} # get the workflow config workflow = get_workspace_config(WORKSPACE_NAMESPACE, WORKSPACE_NAME, WORKFLOW_NAMESPACE, WORKFLOW_NAME, headers) check_fapi_response(workflow, 200) workflow_config_json = workflow.json() # This workflow uses inputs from the data table as well as the file_path # value input to this function. We first pull the root entity type from # the workflow config, and then look for sets of that entity type, # selecting the first set found in the data table. root_entity_type = workflow_config_json['rootEntityType'] expression = f'this.{root_entity_type}s' set_entity_type = f'{root_entity_type}_set' entities = get_entities(WORKSPACE_NAMESPACE, WORKSPACE_NAME, set_entity_type, headers) check_fapi_response(entities, 200) all_set_names = [ent['name'] for ent in entities.json()] set_to_use = all_set_names[0] # use the first set # Next we need to add the specific input from file_path. We update this value # in the inputs section of the workflow_config_json. for input_value in workflow_config_json['inputs']: if input_value.endswith(INPUT_NAME): workflow_config_json['inputs'][input_value] = f"\"{file_path}\"" # remove outputs assignment from config workflow_config_json['outputs'] = {} # update the workflow configuration updated_workflow = update_workspace_config(WORKSPACE_NAMESPACE, WORKSPACE_NAME, WORKFLOW_NAMESPACE, WORKFLOW_NAME, workflow_config_json, headers) check_fapi_response(updated_workflow, 200) # launch the workflow create_submisson_response = create_submission(WORKSPACE_NAMESPACE, WORKSPACE_NAME, WORKFLOW_NAMESPACE, WORKFLOW_NAME, headers, use_callcache=True, entity=set_to_use, etype=set_entity_type, expression=expression) check_fapi_response(create_submisson_response, 201) submission_id = create_submisson_response.json()['submissionId'] print(f"Successfully created submission: submissionId = {submission_id}.")
def __init__(self, name, url): normalized_name = unicodedata.normalize('NFKD', name).encode('ascii', 'ignore') if normalized_name == 'All': # no particular topics url_str = url.string.encode('ascii', 'ignore') else: gn_rss = 'https://www.google.com/news?cf=all&ned=us&hl=en&output=rss&num=100' url_str = gn_rss + '&q=' + urllib.quote(normalized_name) self._Article = None # inner class self._name = normalized_name self._url = url_str self._entities = get_entities(self)
def main(): title = sys.argv[1] client = HttpClient() params = { 'action': 'query', 'format': 'json', 'prop': 'revisions', 'rvprop': 'content', 'titles': title, } data = client.get(URL, params) if data is not None: contents = get_contents(data) entities = get_entities(contents) pprint(entities) print('=' * 72) print('\nカテゴリのみを出力') for category in filter(lambda x: x.startswith('Category:'), entities): print(category)
def __load_files(self): logging.info("Started loading the files...") with open(self.ans_file, "r") as afile: self.avals = pkl.load(afile) with open(self.desc_file, "r") as dfile: self.dvals = pkl.load(dfile) with open(self.q_file, "r") as qfile: self.qvals = pkl.load(qfile) with open(self.vocab, "r") as vocabf: self.adict = pkl.load(vocabf) self.iadict = {v: k for k, v in self.adict.iteritems()} self.data_len = len(self.avals) if self.stop == -1: self.stop = self.data_len self.vsize = len(self.adict) self.entities = get_entities(self.adict) logging.info("Loaded the files...")
def __load_files(self): logging.info("Started loading the files...") with open(self.ans_file, "r") as afile: self.avals = pkl.load(afile) with open(self.desc_file, "r") as dfile: self.dvals = pkl.load(dfile) with open(self.q_file, "r") as qfile: self.qvals = pkl.load(qfile) with open(self.vocab, "r") as vocabf: self.adict = pkl.load(vocabf) self.iadict = {v:k for k, v in self.adict.iteritems()} self.data_len = len(self.avals) if self.stop == -1: self.stop = self.data_len self.vsize = len(self.adict) self.entities = get_entities(self.adict) logging.info("Loaded the files...")
def test_single_entry(self): expected = ['エントリ'] actual = get_entities("括弧でくくられた[[エントリ]]を取り出す") self.assertEqual(expected, actual)
def main(): parser = argparse.ArgumentParser( description="""Count comments in a given number of posts""") parser.add_argument( '-c', '--conf', type=str, metavar='', required=True, help='Specify the path of the configuration file') args = parser.parse_args() config_path = args.conf start = time.time() logger = get_logger(__name__) logger.setLevel(logging.DEBUG) conf = load_config(config_path) supported_languages = ["it", "en"] lang = input("Insert language (it, en): ") if lang not in supported_languages: logger.error("Please provide a valid language. Supported: 'en', 'it'") sys.exit(1) else: try: model = conf.get(lang) nlp = spacy.load(model) except OSError: logger.error("Could not find model in conf file. Please double check") sys.exit(0) n_posts = check_n_posts() if not n_posts.isdigit() and n_posts != "-1": logger.error("Please give a number. Exiting") sys.exit(0) try: access_token = conf["access_token"] page_id = conf["page_id"] n_top_entities = conf["n_top_entities"] data_dir_path = os.path.join(page_id, conf["data_dir_name"]) data_filename = "{}_{}.tsv".format(conf["data_entities_prefix"], str(n_posts)) plots_dir_path = os.path.join(page_id, conf["plots_dir_name"]) barplot_filename = "{}_{}posts_ner.png".format(conf["barplot_filename"], str(n_posts)) barplot_filepath = os.path.join(plots_dir_path, barplot_filename) except KeyError: logger.error( "Invalid configuration file. Please check template and retry") sys.exit(0) try: graph = facebook.GraphAPI(access_token) logger.info("Graph API connected") profile = graph.get_object(page_id) except facebook.GraphAPIError as e: logger.error("Could not log in. {}".format(e)) sys.exit(0) if n_posts != "": logger.info("Getting the last {} posts".format(n_posts)) else: logger.warning( "Requesting posts with no limits. " "This could be susceptible of limitations" " in the near future due to high rate" ) local_start = time.time() posts = graph.get_connections(profile["id"], "posts", limit=n_posts) comments = [] for post in posts["data"]: url_post = "https://www.facebook.com/posts/{}".format(post["id"]) logger.info("Getting data for post {}".format(url_post)) post_data = get_post_data(access_token, post["id"]) post_comments = get_comments(post_data) if len(post_comments) == 0: logger.warning( """Apparently, there are no comments at the selected post Check the actual post on its Facebook page https://www.facebook.com/posts/{}""".format(post["id"]) ) comments.extend(post_comments) if len(comments) == 0: logger.error("Could not get any comments. Exiting gracefully") sys.exit(0) elif len(comments) < 100: logger.warning( "Found {} comment(s). Not enough data " "to make much sense. Plots will be made regardless".format( len(comments) ) ) else: logger.info("Got {} comments from {} post(s) in {} seconds".format( len(comments), len(posts["data"]), round((time.time() - local_start), 1))) local_start = time.time() entities = [] for comment in comments: ents = get_entities(nlp, comment) entities.extend(ents) logger.info("Extracted {} entities out of {} comments in {} seconds".format( len(entities), len(comments), round((time.time() - local_start), 2))) entities_data = count_entities(entities) create_nonexistent_dir(data_dir_path) data_filepath = os.path.join(data_dir_path, data_filename) columns = ["entities", "count"] data_to_tsv(entities_data, columns, data_filepath) logger.info("Saved {} unique entities and their counts in {} ".format( len(entities_data), data_filepath)) create_nonexistent_dir(plots_dir_path) plot_labels = ["Entities", "Counts"] save_barplot(entities_data, plot_labels, n_top_entities, barplot_filepath, type_="entities") logger.info("Bar plot saved at {}".format(barplot_filepath)) logger.info("\a\a\aDIN DONE! in {} seconds".format( round((time.time() - start), 1)))
import os, json from utils import get_entities from collections import OrderedDict def list2dict(l): _dict = {} for key in l: _dict[key] = [key] return _dict def get_dict(original_dict): new_dict = OrderedDict() for key, values in original_dict.items(): new_dict[key] = list2dict(values) return new_dict def dict2json(d, out_file): with open(out_file, 'w') as fp: json.dump(d, fp) entity_dict = get_entities('entities.txt') dd = get_dict(entity_dict) file_name = 'deepPavlov/DeepPavlov/download/slots/pharma.json' output_file = os.path.join(os.path.join(os.getcwd(), os.pardir), file_name) dict2json(dd, output_file)
def main(): service = authenticate(sys.argv) calendars = get_existing_calendars(service) # make_calendars_public(service, calendars) entities = get_entities() create_calendars_for_entities(entities, calendars, service)
def test_single_entry(): expected = ['エントリ'] actual = get_entities("括弧でくくられた[[エントリ]]を取り出す") assert expected == actual
def test_empty_string(self): self.assertEqual([], get_entities(""))
print('Test Acc: {:.3f}% ({}/{})'.format(100 * correct / total, correct, total)) def simulate_test_dialogs(how_many, print_simulated_dialog=False): #model.eval() with torch.no_grad(): for i in range(how_many): episode_actions, episode_return, dialog = simulate_dialog( system_acts, is_test=True) if print_simulated_dialog: print(dialog, 'return', episode_return) return episode_return entities = get_entities('dialog-bAbI-tasks/dialog-babi-kb-all.txt') for idx, (ent_name, ent_vals) in enumerate(entities.items()): print('entities', idx, ent_name, ent_vals[0]) assert args.task == 5 or args.task == 6, 'task must be 5 or 6' if args.task == 5: fpath_train = 'dialog-bAbI-tasks/dialog-babi-task5-full-dialogs-trn.txt' #fpath_train = 'pretrain_dialogs.txt' fpath_test = 'dialog-bAbI-tasks/dialog-babi-task5-full-dialogs-tst-OOV.txt' elif args.task == 6: # this is not working yet fpath_train = 'dialog-bAbI-tasks/dialog-babi-task6-dstc2-trn.txt' fpath_test = 'dialog-bAbI-tasks/dialog-babi-task6-dstc2-tst.txt' system_acts = [g.SILENT] vocab = []
def main(): parser = argparse.ArgumentParser( description="""Count comments in a given number of posts""") parser.add_argument('-c', '--conf', type=str, metavar='', required=True, help='Specify the path of the configuration file') args = parser.parse_args() config_path = args.conf start = time.time() logger = get_logger(__name__) logger.setLevel(logging.DEBUG) conf = load_config(config_path) supported_languages = ["it", "en"] lang = input("Insert language (it, en): ") if lang not in supported_languages: logger.error("Please provide a valid language. Supported: 'en', 'it'") sys.exit(1) else: try: model = conf.get(lang) nlp = spacy.load(model) except OSError: logger.error( "Could not find model in conf file. Please double check") sys.exit(0) post_id = "" while post_id == "": post_id = input("Provide post ID: ") try: access_token = conf["access_token"] page_id = conf["page_id"] n_top_entities = conf["n_top_entities"] data_dir_path = os.path.join(page_id, conf["data_dir_name"]) data_filename = "{}_{}{}".format(conf["data_entities_prefix"], post_id, ".csv") plots_dir_path = os.path.join(page_id, conf["plots_dir_name"]) barplot_filename = "{}_{}{}".format(conf["barplot_filename"], post_id, "_ner.png") barplot_filepath = os.path.join(plots_dir_path, barplot_filename) except KeyError: logger.error( "Invalid configuration file. Please check template and retry") sys.exit(0) actual_post_id = page_id + "_" + post_id url_post = "https://www.facebook.com/posts/{}".format(actual_post_id) logger.info("Getting data for post {}".format(url_post)) local_start = time.time() data = get_post_data(access_token, actual_post_id) comments = get_comments(data) if len(comments) == 0: logger.error("""Apparently, there are no comments at the selected post Check the actual post on its Facebook page https://www.facebook.com/{}/posts/{}""".format(page_id, post_id)) sys.exit(0) elif len(comments) < 100: logger.warning( "Got {} comments. Not enough data " "to make much sense. Plots will be made regardless".format( len(comments))) else: logger.info("Got {} comments in {} seconds".format( len(comments), round((time.time() - local_start), 2))) local_start = time.time() entities = [] for comment in comments: ents = get_entities(nlp, comment) entities.extend(ents) logger.info( "Extracted {} entities out of {} comments in {} seconds".format( len(entities), len(comments), round((time.time() - local_start), 2))) entities_data = count_entities(entities) create_nonexistent_dir(data_dir_path) data_filepath = os.path.join(data_dir_path, data_filename) columns = ["entities", "count"] data_to_tsv(entities_data, columns, data_filepath) logger.info("Saved {} unique entities and their counts in {} ".format( len(entities_data), data_filepath)) create_nonexistent_dir(plots_dir_path) plot_labels = ["Entities", "Counts"] save_barplot(entities_data, plot_labels, n_top_entities, barplot_filepath) logger.info("Bar plot saved at {}".format(barplot_filepath)) logger.info("\a\a\aDIN DONE! in {} seconds".format( round((time.time() - start), 1)))
import re from utils import get_pdf_name_list, get_entities, es_search if __name__ == "__main__": pdfs = get_pdf_name_list() entities = get_entities() filter_list = [] for k, v in entities.items(): filter_list = filter_list + v must_list = [{ "match_phrase": { "name": "三生国健药业(上海)股份有限公司科创板首次公开发行股票招股说明书(上会稿).pdf" } }] # for k, v in entities.items(): # must_list = must_list + [{"match": {"content": f}} for f in v] filter_list = [{"term": {"content": f}} for f in filter_list] result = es_search(must_list, [], [], []) # print(result["hits"])
def test_none_param(self): with self.assertRaises(TypeError): get_entities(None)
def test_actual_contents(self): actual = get_entities(self.contents) self.assertEqual(223, len(actual)) self.assertEqual('File:Prog_one.png', actual[0]) self.assertEqual('Category:ソフトウェア開発工程', actual[-1])
parser.add_argument('--task', type=int, default=5, help='5 for Task 5 and 6 for Task 6') parser.add_argument('--seed', type=int, default=1111, help='random seed') parser.add_argument('--vocab_full', type=int, default=0, help='option to use full embedding matrix') args = parser.parse_args() # Set the random seed manually for reproducibility. torch.manual_seed(args.seed) # entities = get_entities('/Users/graeme/dialog-bAbI-tasks/dialog-babi-kb-all.txt') entities = get_entities('entities.txt') # for idx, (ent_name, ent_vals) in enumerate(entities.items()): # print('entities', idx, ent_name) # assert args.task == 5 or args.task == 6, 'task must be 5 or 6' # if args.task == 5: fpath_train = 'dialogues_train.txt' fpath_test = 'dialogues_test.txt' save_path = 'new_trained_model.pt' # word2vec_path = 'faqmodel.bin' word2vec_path = '/Users/mac/Downloads/GoogleNews-vectors-negative300.bin' database = 'NovartisQA' # embedding_file = 'faqmodel.pkl' embedding_file = 'pre_embd_w_full.pickle'
def show_form(text): data = get_entities(text) return render_template("finalform.html", data=data)
def test_actual_contents(contents): actual = get_entities(contents) assert 223 == len(actual) assert 'File:Prog_one.png' == actual[0] assert 'Category:ソフトウェア開発工程' == actual[-1]