def ana_results_save(self): self.table.selectAll() items = self.table.selectedItems() deleted_keywords = {} modified_keywords = {} new_keywords = {} # TODO gérer la modification du texte for item in items: item_row = item.row() item_column = item.column() equiv_key = "(" + str(item_row) + "," + str(item_column) + ")" key = self.equivalence_table[equiv_key] if item.checkState() == Qt.Unchecked: # Mots-clés décochés deleted_keywords[key] = {"modification": "supprime"} elif item.checkState() == Qt.Checked and item.text() != key: # Mots-cles modifiés modified_keywords[key] = {"modification": "modifie", "nouveau_mot_cle": item.text()} added_keywords = self.ana_results_add_keyword.text().split(";") for new_keyword in added_keywords: new_keywords[new_keyword.strip()] = {"modification": "ajoute"} modified_keywords.update(deleted_keywords) modified_keywords.update(new_keywords) with open(self.project_directory + "/output/modified_keywords.json", "w", encoding="utf8") as outfile: json.dump(modified_keywords, outfile, ensure_ascii=False, indent=4) outfile.close() self.ana_results_saved.setText("Résultats sauvegardés")
def export_json(): print('Exporting json') # organization json file bulk = [] entries = Organization.objects.all() for e in entries: x = {'Organization': e.name} bulk.append(x) with io.open('./cos/static/toporgs.json', 'w', encoding='utf8') as \ data_file: json.dump(bulk, data_file, indent=1, ensure_ascii=False) print('Finished exporting toporgs.json') # journal first json file save_to_journal_json('rrjournals', 'registered', 1) # second json file save_to_journal_json('rrjournalssome', 'featured', 1) # third json file save_to_journal_json('rrjournalsspecial', 'special', 1) # fourth json file save_to_journal_json('preregjournals', 'preregistered', 2) # fifth json file save_to_journal_json('topjournals', 'top', 2) print('Finished exporting journal json files') print("Completed exporting json")
def p4(): cur.execute("SELECT year, Sex, Marital_Status, Age_Value FROM mortality") i = 0 while True: try: data = cur.fetchone() i+=1 if i%500000 == 0: print i if int(data[0]) not in YSMA.keys(): YSMA[int(data[0])] = {str(data[1]): {str(data[2]): [int(data[3])]}} elif str(data[1]) not in YSMA[int(data[0])]: YSMA[int(data[0])][str(data[1])] = {str(data[2]): [int(data[3])]} elif str(data[2]) not in YSMA[int(data[0])][str(data[1])]: YSMA[int(data[0])][str(data[1])][str(data[2])] = [int(data[3])] else: YSMA[int(data[0])][str(data[1])][str(data[2])].append(int(data[3])) except: break with open('C:\Users\jmorris\Documents\Fall_2015\Data Visualization\HW4/YearSexMarriageAge.json', 'w') as outfile: #Julian's file path json.dump(YSMA, outfile, indent = 4) return YSMA
def save_settings_export(properties): settings = { "option_export_scene" : properties.option_export_scene, "option_embed_meshes" : properties.option_embed_meshes, "option_url_base_html" : properties.option_url_base_html, "option_copy_textures" : properties.option_copy_textures, "option_lights" : properties.option_lights, "option_cameras" : properties.option_cameras, "option_animation" : properties.option_animation, "option_frame_step" : properties.option_frame_step, "option_all_meshes" : properties.option_all_meshes, "option_flip_yz" : properties.option_flip_yz, "option_materials" : properties.option_materials, "option_normals" : properties.option_normals, "option_colors" : properties.option_colors, "option_uv_coords" : properties.option_uv_coords, "option_edges" : properties.option_edges, "option_faces" : properties.option_faces, "option_vertices" : properties.option_vertices, "option_vertices_truncate" : properties.option_vertices_truncate, "option_scale" : properties.option_scale, "align_model" : properties.align_model } fname = get_settings_fullpath() f = open(fname, "w") json.dump(settings, f)
def copy_json(input_path, output_path): with open(input_path) as input: with open(output_path, "w+") as output: json.dump( json.load(input), output, indent=2)
def body(self): if self.playback: subscription_id = MOCKED_SUBSCRIPTION_ID else: subscription_id = self.cmd('account list --query "[?isDefault].id" -o tsv') role_name = 'cli-test-role' template = { "Name": "Contoso On-call", "Description": "Can monitor compute, network and storage, and restart virtual machines", "Actions": ["Microsoft.Compute/*/read", "Microsoft.Compute/virtualMachines/start/action", "Microsoft.Compute/virtualMachines/restart/action", "Microsoft.Network/*/read", "Microsoft.Storage/*/read", "Microsoft.Authorization/*/read", "Microsoft.Resources/subscriptions/resourceGroups/read", "Microsoft.Resources/subscriptions/resourceGroups/resources/read", "Microsoft.Insights/alertRules/*", "Microsoft.Support/*"], "AssignableScopes": ["/subscriptions/{}".format(subscription_id)] } template['Name'] = role_name _, temp_file = tempfile.mkstemp() with open(temp_file, 'w') as f: json.dump(template, f) role = self.cmd('role create --role-definition {}'.format(temp_file.replace('\\', '\\\\')), None) self.cmd('role list -n {}'.format(role_name), checks=[JMESPathCheck('[0].properties.roleName', role_name)]) self.cmd('role delete -n {}'.format(role_name), None) self.cmd('role list -n {}'.format(role_name), NoneCheck())
def main(): api_url, username, password = get_account_data(True) backup_files = get_nsbackup_files(True) if 'pickle_backup_file' in backup_files.keys(): from pickle import Pickler if 'json_backup_file' in backup_files.keys(): import json # Instantiate the inwx class (does not connect yet but dispatches calls to domrobot objects with the correct API URL inwx_conn = domrobot(api_url, username, password, 'en', False) # get the list of all domains: domains = inwx_conn.nameserver.list()['domains'] # get all the nameserver entries for each domain current, total = 0, len(domains) nsentries = dict() for domain in domains: current += 1 domain = domain['domain'] print "%i of %i - Currently backing up %s." % (current, total, domain) nsentries[domain] = inwx_conn.nameserver.info({'domain': domain})['record'] if 'pickle_backup_file' in backup_files.keys(): Pickler(open(backup_files['pickle_backup_file'],'wb')).dump(nsentries) print "Wrote backup file using Python Module Pickle : %s." % backup_files['pickle_backup_file'] if 'json_backup_file' in backup_files.keys(): json.dump(nsentries, open(backup_files['json_backup_file'], 'w')) print "Wrote backup file using Python Module JSON: %s." % backup_files['json_backup_file']
def write(self, rootdir): """ Write the metadata to a JSON file in the rootdir. Values contained in the metadata are: first_trading_day : string 'YYYY-MM-DD' formatted representation of the first trading day available in the dataset. minute_index : list of integers nanosecond integer representation of the minutes, the enumeration of which corresponds to the values in each bcolz carray. ohlc_ratio : int The factor by which the pricing data is multiplied so that the float data can be stored as an integer. """ metadata = { 'first_trading_day': str(self.first_trading_day.date()), 'minute_index': self.minute_index.asi8.tolist(), 'market_opens': self.market_opens.values. astype('datetime64[m]'). astype(int).tolist(), 'market_closes': self.market_closes.values. astype('datetime64[m]'). astype(int).tolist(), 'ohlc_ratio': self.ohlc_ratio, } with open(self.metadata_path(rootdir), 'w+') as fp: json.dump(metadata, fp)
def auth_dialog(self): try: with open(CONF_FILE) as conf: token = json.load(conf) except (OSError, json.JSONDecodeError, KeyError): print("Your token file doesn't exist or is malformed.") print("If you WANT to proceed anonymously, pass" "`-a/--anonymous` on the command line.") print("Or you can create a new token now. " "Create token? [y/n] ", end='') if input().lower() in ('y', 'yes'): username = input('Username: '******'gist3_{}'.format(id_gen()) resp = req.post(API_BASE + '/authorizations', auth=(username, password), json={'scopes': ['gist'], 'note': note}) if resp.status_code == req.codes.created: token = resp.json()['token'] with open(CONF_FILE, 'w') as conf: json.dump(token, conf) print('Token created & saved to {}'.format(CONF_FILE)) else: print('There was an error from github: ') print(json.dumps(resp.json(), sort_keys=True, indent=4)) sys.exit(2) else: print('Aborting...') sys.exit(0) return AccessTokenAuth(token)
def delete(self, thema, id, beitragID=None): discussionpath = "./data/themen/" + thema + "/" + id + ".json" with open(discussionpath, "r") as discussionfile: discussion = json.load(discussionfile) if beitragID == None: if discussion["Status"] == "deleted": discussion["Status"] = " " else: discussion["Status"] = "deleted" discussion["Bearbeiter"] = cherrypy.session["Benutzername"] discussion["Bearbeitet"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) else: for post in discussion["Beitraege"]: if post["ID"] == beitragID: if post["Status"] == "deleted": post["Status"] = " " else: post["Status"] = "deleted" post["Bearbeiter"] = cherrypy.session["Benutzername"] post["Bearbeitet"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) with open(discussionpath, "w") as discussionfile: json.dump(discussion, discussionfile, indent=4)
def get_opendata2_courses(): good_courses = 0 file_name = os.path.join(os.path.realpath(os.path.dirname(__file__)), '%s/opendata2_departments.json' % c.DEPARTMENTS_DATA_DIR) with open(file_name) as departments_file: departments = json.load(departments_file) # Create a text file for every department for d in departments: department = d['subject'] open_data_json = requests.get( 'https://api.uwaterloo.ca/v2/courses/{0}.json?key={1}'.format( department.upper(), s.OPEN_DATA_API_KEY)).json open_data_catalog_numbers = [] for course in open_data_json['data']: open_data_catalog_numbers.append(course['catalog_number']) # We now poll the individual endpoints of each course for the data current_dep_json = [] course_url = 'https://api.uwaterloo.ca/v2/courses/{0}/{1}.json?key={2}' for course in open_data_catalog_numbers: good_courses += 1 json_data = requests.get(course_url.format(department.upper(), course, s.OPEN_DATA_API_KEY)).json current_dep_json.append(json_data['data']) out_file_name = os.path.join( os.path.realpath(os.path.dirname(__file__)), 'opendata2_courses/%s.json' % department.lower()) with open(out_file_name, 'w') as courses_out: json.dump(current_dep_json, courses_out) print 'Found {num} good courses'.format(num=good_courses)
def main(): # THIS CODE IS RUN ONE TIME ONLY (#SAVETHESERVERS) # For Champions: #api = RiotAPI('3e888957-13a3-4ba2-901c-fae3e421d998') #r = api.get_championList() #with open('testChampion.json', 'w') as outfile: # json.dump(r, outfile, indent=4) # -------------------------------------------------------- with open('testChampion.json', 'r') as data_file: data = json.load(data_file) championIds = [data['data'][championName]['id'] for championName in data['data']] championNames = [championName for championName in data['data']] championTitles = [data['data'][championName]['title'] for championName in data['data']] championReference = dict(zip(championIds, zip(championNames, championTitles))) with open('championReference.json', 'w') as outfile: json.dump(championReference, outfile, indent=4)
def do_deployment_manifest(system, configuration): app.log(system, "Creating deployment manifest in", system['sandbox']) data = {'configuration': configuration} metafile = os.path.join(system['sandbox'], 'baserock', 'deployment.meta') with app.chdir(system['sandbox']), open(metafile, "w") as f: json.dump(data, f, indent=4, sort_keys=True, encoding='unicode-escape') f.flush()
def _update_manifest(self,dom_id,ts_esmf,var,kv): """ Adds a key-value set to the dictionary storing metadata for time ts_esmf and variable var. :param dom_id: the domain id (1, 2, 3, ...) :param ts_esmf: ESMF time string :param var: variable name :param kv: key-value dictionary to merge """ # update the manifest with the domain/ts_esmf/var info dom = self.manifest.get(str(dom_id), {}) self.manifest[str(dom_id)] = dom # extract timestamp td = dom.get(ts_esmf, {}) dom[ts_esmf] = td # store variable in timestamp vd = td.get(var, {}) td[var] = vd vd.update(kv) mf_path = os.path.join(self.output_path, self.product_name + '.json') # synchronize the file json.dump(self.manifest, open(mf_path, 'w'))
def dump_app(id, **kw): # Because @robhudson told me to. from mkt.api.resources import AppResource # Note: not using storage because all these operations should be local. target_dir = os.path.join(settings.DUMPED_APPS_PATH, 'apps', str(id / 1000)) target_file = os.path.join(target_dir, str(id) + '.json') try: obj = Webapp.objects.get(pk=id) except Webapp.DoesNotExist: task_log.info(u'Webapp does not exist: {0}'.format(id)) return req = RequestFactory().get('/') req.user = AnonymousUser() req.REGION = WORLDWIDE if not os.path.exists(target_dir): os.makedirs(target_dir) task_log.info('Dumping app {0} to {1}'.format(id, target_file)) res = AppResource().dehydrate_objects([obj], request=req) json.dump(res[0], open(target_file, 'w'), cls=JSONEncoder) return target_file
def saveTweets(self): meaningful = self.jsonAccepted*self.cfg['KeepAccepted'] + self.jsonPartial*self.cfg['KeepPartial'] + self.jsonExcluded*self.cfg['KeepExcluded'] if len(meaningful)>1: print "\nDumping tweets to file, contains %s tweets with %s accepted, %s rejected, %s partial matches, and %s irrelevant" % (len(meaningful), self.acceptedCount, self.excludedCount, self.partialCount, self.irrelevantCount) if self.cfg['TweetData'] != 'all': meaningful = cleanJson(meaningful,self.cfg,self.tweetTypes) #timeStamp = datetime.date.today().strftime("%A") timeStamp = self.startTime self.lastWrite = self.startDay if self.cfg['KeepRaw']: with open(self.pathOut+'Raw_'+self.cfg['FileName']+'_'+timeStamp+'.json', 'w') as outFile: json.dump(self.jsonRaw,outFile) outFile.close() with open(self.pathOut+'FilteredTweets_'+self.cfg['FileName']+'_'+timeStamp+'.json', 'w') as outFile: json.dump(meaningful,outFile) outFile.close() print 'Json text dump complete, buffering....' time.sleep(1) giSeeker.flushTweets(self) else: print "No tweets found for date" print "Updating geoPickle" updateGeoPickle(self.geoCache,self.cfg['Directory']+pickleName)
def saveTweets(self): print "\nDumping tweets to file, contains %s tweets with %s accepted, %s rejected, %s partial matches, and %s irrelevant" % (self.cfg['StopCount'], self.acceptedCount, self.excludedCount, self.partialCount, self.irrelevantCount) print '\tJson text dump complete....\n' meaningful = self.jsonAccepted*self.cfg['KeepAccepted'] + self.jsonPartial*self.cfg['KeepPartial'] + self.jsonExcluded*self.cfg['KeepExcluded'] if self.cfg['TweetData'] != 'all': meaningful = cleanJson(meaningful,self.cfg,self.tweetTypes) timeStamp = self.startTime if self.cfg['KeepRaw']: with open(self.pathOut+'Raw_'+self.cfg['FileName']+'_'+timeStamp+'.json', 'w') as outFile: json.dump(self.jsonRaw,outFile) outFile.close() with open(self.pathOut+'FilteredTweets_'+self.cfg['FileName']+'_'+timeStamp+'.json', 'w') as outFile: json.dump(meaningful,outFile) outFile.close() giListener.flushTweets(self) print "Updating geoPickle" updateGeoPickle(self.geoCache,self.cfg['Directory']+pickleName)
def renewEventStagerStatus(self): canFinish_file = os.path.join(self.__workDir, 'EventStagerStatusCan.json') finished_file = os.path.join(self.__workDir, 'EventStagerStatus.json') if self.__isDaemon: if os.path.exists(canFinish_file): #with open(canFinish_file) as inputFile: # origin_status = json.load(inputFile) # self.__canFinish = origin_status['canFinish'] self.__canFinish = True if self.__status == "finished": status = {'status': self.__status} with open(finished_file, 'w') as outputFile: json.dump(status, outputFile) elif os.path.exists(finished_file): os.remove(finished_file) else: if os.path.exists(finished_file): #with open(finished_file) as inputFile: # origin_status = json.load(inputFile) # self.__status = origin_status['status'] self.__status = "finished" if self.__canFinish: status = {'canFinish': self.__canFinish} with open(canFinish_file, 'w') as outputFile: json.dump(status, outputFile) elif os.path.exists(canFinish_file): os.remove(canFinish_file)
def parse_duration_ffmpeg(media_url): try: with open(_MEDIA_FILE) as f: _cache = json.load(f) except IOError: _cache = {} except ValueError: # the json file is corrupted _cache = {} if media_url not in _cache: command = ['ffmpeg', '-i', media_url] out, err = wrap_subprocess(command) REGEX = re.compile('Duration: (\d+):(\d+):(\d+).(\d+)') matches = REGEX.findall(err) # if matches: try: found, = matches except ValueError: print err return hours = int(found[0]) minutes = int(found[1]) minutes += hours * 60 seconds = int(found[2]) seconds += minutes * 60 duration = seconds + minutes * 60 + hours * 60 * 60 _cache[media_url] = duration with open(_MEDIA_FILE, 'w') as f: json.dump(_cache, f, indent=4) return _cache[media_url]
def main(argv): global result, g_debug parser = argparse.ArgumentParser(description="Convert LDIF file to json") parser.add_argument("--input", help="input file", required=True) parser.add_argument("--country_code", help="country code, e.g. +41", required=True) parser.add_argument("--merge", help="file to merge with", default="out.json") parser.add_argument('--debug', action='store_true') args = parser.parse_args() g_debug = args.debug name = os.path.splitext(os.path.basename(args.merge))[0] result = [] # merge try: data = open(args.merge, "r").read() j = json.loads(data) name = j["name"] result = j["entries"] #debug(result) except IOError: pass # convert parser = MyLDIF(open(args.input, "r")) parser.parse() result = cleanup_entries(result, args.country_code) if len(result) != 0: data = OrderedDict(( ("name", name), ("entries", result) )) with open(args.merge, 'w') as outfile: json.dump(data, outfile, indent=2)
def _flatten_samples(samples, base_file): """Create a flattened JSON representation of data from the bcbio world map. """ out_file = "%s-samples.json" % utils.splitext_plus(base_file)[0] flat_data = [] for data in samples: cur_flat = {} for key_path in [["analysis"], ["description"], ["rgnames"], ["config", "algorithm"], ["metadata"], ["genome_build"], ["resources"], ["files"], ["reference"], ["genome_resources"], ["vrn_file"]]: cur_key = "__".join(key_path) for flat_key, flat_val in _to_cwldata(cur_key, tz.get_in(key_path, data)): cur_flat[flat_key] = flat_val flat_data.append(cur_flat) out = {} for key in sorted(list(set(reduce(operator.add, [d.keys() for d in flat_data])))): out[key] = [] for cur_flat in flat_data: out[key].append(cur_flat.get(key)) # special case for back-compatibility with fasta specifications -- yuck if "reference__fasta__base" not in out and "reference__fasta" in out: out["reference__fasta__base"] = out["reference__fasta"] del out["reference__fasta"] with open(out_file, "w") as out_handle: json.dump(out, out_handle, sort_keys=True, indent=4, separators=(',', ': ')) return out_file, _samplejson_to_inputs(out), out
def run(self): self.correct_octave_errors_by_chunks() self.remove_extreme_values() self.correct_jumps() self.pitch = list(reversed(self.pitch)) self.correct_jumps() self.pitch = list(reversed(self.pitch)) self.filter_noise_region() self.correct_oct_error() self.pitch = list(reversed(self.pitch)) self.correct_oct_error() self.pitch = list(reversed(self.pitch)) self.correct_octave_errors_by_chunks() self.filter_chunks_by_energy(chunk_limit=60) if self.out: self.data['pitch'] = self.pitch with open(self.fname[:-5] + "_filtered.json", 'w') as f: json.dump(self.data, f) return self.pitch
def SaveScene(file, quad_tree, extra_data=None): export_data = [] if extra_data: for item in extra_data: export_data.append(item) for node in quad_tree: for child in node.getChildren(): temp = {} if child.hasPythonTag("actor_files"): temp["actor"] = unicode(child.getPythonTag("actor_files")[0]) temp["actor_anims"] = child.getPythonTag("actor_files")[1] temp["actor_collision"] = child.getPythonTag("actor_files")[2] elif child.hasPythonTag("model_file"): temp["model"] = unicode(child.getPythonTag("model_file")) if child.hasPythonTag("light_color"): c = child.getPythonTag("light_color") temp["color_r"] = c[0] temp["color_g"] = c[1] temp["color_b"] = c[2] temp["rotation_h"] = child.getH(render) temp["rotation_p"] = child.getP(render) temp["rotation_r"] = child.getR(render) temp["position_x"] = child.getX(render) temp["position_y"] = child.getY(render) temp["position_z"] = child.getZ(render) temp["scale"] = child.getScale()[0] temp["parent_name"] = node.getName() temp["parent_index"] = quad_tree.index(node) temp["props"] = unicode(child.getPythonTag("props")) export_data.append(temp) with open(file, "w") as outfile: json.dump(export_data, outfile, indent=4, separators=(",", ": "), sort_keys=True)
def dump_results(specs, output_path): with open(output_path, 'w') as output_file: results = OrderedDict([ ("env", ENV), ("specs", specs) ]) json.dump(results, output_file, indent=4)
def main(): my_config = get_config() new_synapse_config = generate_configuration( my_config, get_zookeeper_topology(), get_all_namespaces() ) with tempfile.NamedTemporaryFile() as tmp_file: new_synapse_config_path = tmp_file.name with open(new_synapse_config_path, 'w') as fp: json.dump(new_synapse_config, fp, sort_keys=True, indent=4, separators=(',', ': ')) # Match permissions that puppet expects os.chmod(new_synapse_config_path, 0644) # Restart synapse if the config files differ should_restart = not filecmp.cmp(new_synapse_config_path, my_config['config_file']) # Always swap new config file into place. Our monitoring system # checks the config['config_file'] file age to ensure that it is # continually being updated. shutil.copy(new_synapse_config_path, my_config['config_file']) if should_restart: subprocess.check_call(SYNAPSE_RESTART_COMMAND)
def get_filenames_with_industry(link): listed_company={} html=requests.get(link) soup1=BeautifulSoup(html.content) try: with open("url_industry_list.json","r") as outfile: listed_company=json.load(outfile) outfile.close() except: print "file : url_industry_list does not exist" with open("url_industry_list.json","w") as outfile: for a in soup1.findAll('td'): if a.findChild(): l=str(a.findChild().get('href')) if not nasdaq.match(l) and linkPattern.match(l): url=str(a.findChild().get('href')) filname=url.replace('http://','') filname=filname.replace( 'https://','') filname=filname.replace('www.','') # filname=filname.replace('.com','') # filname=filname.replace( '.org','') # filname=filname.replace( '.gov','') filname=re.sub('\..*|\/.*','',filname) ind=str(a.findNextSiblings(style='width:105px')[0].get_text()) #outfile.write(l+'\n') if filname not in listed_company: listed_company[filname]=ind json.dump(listed_company , outfile) #return listed_company print listed_company
def flush(self): file = open(self.path, "wb") json.dump(self.pagesData, file) file.close() for page in self.pages: self.pages[page].flush()
def saveAnnotation(self): f = codecs.open((self.filename + '.info'), 'w', 'utf8') json.dump(self.annotation, f, ensure_ascii=False, indent=4) f.close()
def log_metric(self, name, value, unit=None, global_step=None, extras=None): """Log the benchmark metric information to local file. Currently the logging is done in a synchronized way. This should be updated to log asynchronously. Args: name: string, the name of the metric to log. value: number, the value of the metric. The value will not be logged if it is not a number type. unit: string, the unit of the metric, E.g "image per second". global_step: int, the global_step when the metric is logged. extras: map of string:string, the extra information about the metric. """ if not isinstance(value, numbers.Number): tf.logging.warning( "Metric value to log should be a number. Got %s", type(value)) return with tf.gfile.GFile( os.path.join(self._logging_dir, _METRIC_LOG_FILE_NAME), "a") as f: metric = { "name": name, "value": float(value), "unit": unit, "global_step": global_step, "timestamp": datetime.datetime.now().strftime( _DATE_TIME_FORMAT_PATTERN), "extras": extras} try: json.dump(metric, f) f.write("\n") except (TypeError, ValueError) as e: tf.logging.warning("Failed to dump metric to log file: " "name %s, value %s, error %s", name, value, e)
def json_dump(self, filename="", indent=4, sort_keys=True): if filename == "": filename = "%s.json" % self.instance_name with open(filename, "w") as f: json.dump(self.to_dict(), f, indent=indent, sort_keys=sort_keys)
def cache(running_jobs): # dump to "cache" with open(NOTEBOOK_CACHE, 'w') as fp: json.dump(running_jobs, fp)
def build_new_cnn_relation(train_file, user_feature_file, user_follow_file, out_dir): user_url_dict = {} url_user_dict = {} with open(train_file) as f: lines = f.readlines() for line in lines: uid_no, url_no, freq, ts = line.strip().split() if int(uid_no) not in user_url_dict: user_url_dict[int(uid_no)] = [] user_url_dict[int(uid_no)].append(int(url_no)) if int(url_no) not in url_user_dict: url_user_dict[int(url_no)] = [] url_user_dict[int(url_no)].append(int(uid_no)) print('user url dict: %d, url user dict: %d' % (len(user_url_dict), len(url_user_dict))) user2idx_dict = {} with open(user_feature_file) as f: user_feature_list = json.loads(f.read()) assert len(user_feature_list) == 11576 ##User level: ID, # of followers, # of following, posts urls, post cnt, favorite category, favorite website, uid, name for item in user_feature_list: # item[7] is string user2idx_dict[item[7]] = int(item[0]) uid_mapped_file = '/home/dyou/url_recsys/data_gen/data_181110/uid_mapped.txt' with open(uid_mapped_file) as f: lines = f.readlines() for line in lines: #0 767036004 upayr info = [item.strip() for item in line.strip().split()] assert len(info) == 3 uid_no, uid = info[0], info[1] assert int(uid_no) == user2idx_dict[uid] print('check uid mapped success') with open(user_follow_file) as f: lines = f.readlines() ori_follower_dict = {} ori_follow_in_dict = {} for line in lines[1:]: #follower,followee followee = follow in #757778071734804480,815708250217840640 follower, follow_in = line.strip().split(',') if follower not in ori_follower_dict: ori_follower_dict[follower] = [] if follow_in not in ori_follower_dict[follower]: ori_follower_dict[follower].append(follow_in) if follow_in not in ori_follow_in_dict: ori_follow_in_dict[follow_in] = [] if follower not in ori_follow_in_dict[follow_in]: ori_follow_in_dict[follow_in].append(follower) miss_follower, miss_follow_in = 0, 0 uid_no_follower_dict, uid_no_follow_in_dict = {}, {} for uid in user2idx_dict: if uid not in ori_follower_dict: miss_follower += 1 uid_no_follower_dict[int(user2idx_dict[uid])] = [] else: res_list = [] for uid2 in ori_follower_dict[uid]: if uid2 in user2idx_dict: res_list.append(int(user2idx_dict[uid2])) if len(res_list) == 0: miss_follower += 1 uid_no_follower_dict[int(user2idx_dict[uid])] = [] else: uid_no_follower_dict[int(user2idx_dict[uid])] = res_list if uid not in ori_follow_in_dict: miss_follow_in += 1 uid_no_follow_in_dict[int(user2idx_dict[uid])] = [] else: res_list = [] for uid2 in ori_follow_in_dict[uid]: if uid2 in user2idx_dict: res_list.append(int(user2idx_dict[uid2])) if len(res_list) == 0: miss_follow_in += 1 uid_no_follow_in_dict[int(user2idx_dict[uid])] = [] else: uid_no_follow_in_dict[int(user2idx_dict[uid])] = res_list print('follow users: %d, miss follower: %d, follow in users: %d, miss follow in: %d' % (len(uid_no_follower_dict), miss_follower, len(uid_no_follow_in_dict), miss_follow_in)) url_neighb_urls = {} for url_no in url_user_dict: direct_user_list = url_user_dict[url_no] two_step_url_list = [] for user_no in direct_user_list: two_step_url_list.extend(user_url_dict[user_no]) two_step_url_set = set(two_step_url_list) two_step_url_set.remove(url_no) url_neighb_urls[url_no] = list(two_step_url_set) print('url_neighb_urls:', len(url_neighb_urls)) user_neighb_users = {} for uid_no in user_url_dict: direct_url_list = user_url_dict[uid_no] two_step_user_list = [] for url_no in direct_url_list: two_step_user_list.extend(url_user_dict[url_no]) two_step_user_set = set(two_step_user_list) two_step_user_set.remove(uid_no) user_neighb_users[uid_no] = list(two_step_user_set) print('user_neighb_users:', len(user_neighb_users)) json.dump(user2idx_dict, open(out_dir + os.sep + 'user2idx.json', 'w')) json.dump(ori_follower_dict, open(out_dir + os.sep + 'ori_follower_dict.json', 'w')) json.dump(ori_follow_in_dict, open(out_dir + os.sep + 'ori_follow_in_dict.json', 'w')) json.dump(user_url_dict, open(out_dir + os.sep + 'user_url_dict.json', 'w')) json.dump(url_user_dict, open(out_dir + os.sep + 'url_user_dict.json', 'w')) json.dump(url_neighb_urls, open(out_dir + os.sep + 'url_neighb_urls.json', 'w')) json.dump(user_neighb_users, open(out_dir + os.sep + 'user_neighb_users.json', 'w')) json.dump(uid_no_follower_dict, open(out_dir + os.sep + 'uid_no_follower_dict.json', 'w')) json.dump(uid_no_follow_in_dict, open(out_dir + os.sep + 'uid_no_follow_in_dict.json', 'w')) print('new cnn relation done')
def main(): with open('step1/results1/tagging_1.json') as f: part1=json.load(f) with open('step1/results2/tagging_2.json') as f: part2=json.load(f) with open('step1/results3/tagging_3.json') as f: part3=json.load(f) with open('step1/results4/tagging_4.json') as f: part4=json.load(f) with open('step1/results5/tagging_5.json') as f: part5=json.load(f) all_label=[] with open('../dataset/label_id.txt','r') as f: for line in f.readlines(): label,_ = line.split('\t') all_label.append(label) all_label_set = set(all_label) for file in part1.keys(): label_set = set(part1[file]['result'][0]['labels']) res_label = list(all_label_set - label_set) part1[file]['result'][0]['labels'] += res_label part1[file]['result'][0]['scores'] += ['0']*32 assert len(res_label)==32 assert len(part1[file]['result'][0]['labels'])==82 for file in part2.keys(): label_set = set(part2[file]['result'][0]['labels']) res_label = list(all_label_set - label_set) part2[file]['result'][0]['labels'] += res_label part2[file]['result'][0]['scores'] += ['0']*32 assert len(res_label)==32 assert len(part2[file]['result'][0]['labels'])==82 for file in part3.keys(): label_set = set(part3[file]['result'][0]['labels']) res_label = list(all_label_set - label_set) part3[file]['result'][0]['labels'] += res_label part3[file]['result'][0]['scores'] += ['0']*32 assert len(res_label)==32 assert len(part3[file]['result'][0]['labels'])==82 for file in part4.keys(): label_set = set(part4[file]['result'][0]['labels']) res_label = list(all_label_set - label_set) part4[file]['result'][0]['labels'] += res_label part4[file]['result'][0]['scores'] += ['0']*32 assert len(res_label)==32 assert len(part4[file]['result'][0]['labels'])==82 for file in part5.keys(): label_set = set(part5[file]['result'][0]['labels']) res_label = list(all_label_set - label_set) part5[file]['result'][0]['labels'] += res_label part5[file]['result'][0]['scores'] += ['0']*32 assert len(res_label)==32 assert len(part5[file]['result'][0]['labels'])==82 w1 = 0.2 w2 = 0.2 w3 = 0.2 w4 = 0.2 w5 = 0.2 av_result = {} for file in part1.keys(): av_result[file] = {} av_result[file]['result'] = [] av_result[file]['result'].append({}) av_result[file]['result'][0]['labels'] = [] av_result[file]['result'][0]['scores'] = [] for index, label in enumerate(part1[file]['result'][0]['labels']): av_result[file]['result'][0]['labels'].append(label) index1 = part1[file]['result'][0]['labels'].index(label) index2 = part2[file]['result'][0]['labels'].index(label) index3 = part3[file]['result'][0]['labels'].index(label) index4 = part4[file]['result'][0]['labels'].index(label) index5 = part5[file]['result'][0]['labels'].index(label) score = (w1*eval(part1[file]['result'][0]['scores'][index1]) + w2*eval(part2[file]['result'][0]['scores'][index2]) + w3*eval(part3[file]['result'][0]['scores'][index3]) + w4*eval(part4[file]['result'][0]['scores'][index4]) + w5*eval(part5[file]['result'][0]['scores'][index5]) ) av_result[file]['result'][0]['scores'].append("%.4f" % score) for file in av_result.keys(): score = av_result[file]['result'][0]['scores'] label = av_result[file]['result'][0]['labels'] sorted_score = sorted(enumerate(score), key=lambda x: x[1],reverse=True) idx = [i[0] for i in sorted_score] top20_index=idx[:20] av_result[file]['result'][0]['scores'] =[score[i] for i in top20_index] av_result[file]['result'][0]['labels'] =[label[i] for i in top20_index] with open('step1/results1/train_av_result.json', 'w', encoding="utf-8") as f: json.dump(av_result, f, ensure_ascii=False, indent=4) train_label=pd.read_csv('../dataset/tagging/GroundTruth/tagging_info.txt',header=None,names=['video','label'],sep='\t') predict_df=pd.read_json('step1/results1/train_av_result.json', orient='index') predict_df['label']=predict_df.apply(lambda row:row['result'][0]['labels'],axis=1) predict_df=predict_df.reset_index() predict_df.rename(columns={'index':'video'},inplace=True) predict_df['label_score']=predict_df.apply(lambda row:row['result'][0]['scores'],axis=1) predict_df.drop('result',axis=1,inplace=True) predict_df['soft_label'] = predict_df.apply(lambda row:[0.]*82,axis=1) class Preprocess_label_sparse_to_dense: def __init__(self, index_dict, sep_token=',', is_training=False): self.index_to_tag,self.tag_to_index = extract_dict(index_dict) self.sep_token = sep_token self.is_training = is_training self.max_index = 0 for index in self.index_to_tag: self.max_index = max(index, self.max_index) self.seq_size = self.max_index + 1 self.label_num = self.seq_size def __call__(self, index_str): dense_array = np.zeros(self.seq_size) label_lst = index_str.split(self.sep_token) for label in label_lst: if label in self.tag_to_index: index = self.tag_to_index[label] dense_array[index] = 1.0 return dense_array.astype('float32') def extract_dict(dict_file): index_to_tag = {} tag_to_index = {} for i, line in enumerate(codecs.open(dict_file, 'r', encoding='utf-8')): line = line.strip() if '\t' in line: index, tag = line.split('\t')[:2] elif ' ' in line: index, tag = i, line.rsplit(' ', 1)[0] else: index, tag = i, line try: index = int(index) except: index, tag = int(tag), index index_to_tag[index] = tag tag_to_index[tag] = index return index_to_tag, tag_to_index preprocess_instance=Preprocess_label_sparse_to_dense('../dataset/label_id.txt') tag_to_index = preprocess_instance.tag_to_index predict_df['label_index']=predict_df.apply(lambda row:[tag_to_index[tag] for tag in row['label']],axis=1) for i in range(predict_df.shape[0]): temp_list = np.array(predict_df.iloc[i]['soft_label'],dtype=np.float) temp_list[predict_df.iloc[i]['label_index']] = predict_df.iloc[i]['label_score'] update_row=dict(predict_df.iloc[i]) update_row['soft_label']= temp_list.tolist() predict_df.iloc[i] = pd.Series(update_row) predict_dfV1 = predict_df[['video','soft_label']] predict_dfV1.to_csv('./train_soft_label.csv',index=None)
retriever = DensePassageRetriever(document_store=document_store, query_embedding_model="facebook/dpr-question_encoder-single-nq-base", passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base", use_gpu=True, batch_size=256, embed_title=True) # apparently this is time consuming print("Updating embeddings...'") document_store.update_embeddings(retriever) document_store.save("wiki_dump_embeddings.faiss") # example: # retrieved_doc = retriever.retrieve(query="Why did the revenue increase?") # tune these parameters too print("Running QA Reader") reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=True, no_ans_boost=-10, context_window_size=500, top_k_per_candidate=3, top_k_per_sample=1, num_processes=1, max_seq_len=256, doc_stride=128) # example: # reader.predict(question="Who is the father of Arya Starck?", documents=retrieved_docs, top_k=3) print("Started pipeline") p = Pipeline() p.add_node(component=retriever, name="ESRetriever1", inputs=["Query"]) p.add_node(component=reader, name="QAReader", inputs=["ESRetriever1"]) res = p.run(query="What did Einstein work on?", params={"retriever": {"top_k": 1}}) json.dump(res, open('answer.json', 'w'), default=dumper) exit()
for n in range(min_n,min(max_n + 1, n_original_tokens + 1)): for i in range(n_original_tokens - n + 1): tokens.append(" ".join(original_tokens[i: i + n])) return tokens finalcloud=[] for i in range(len(wlist)): preob = {} preob["daysfromRelease"]=wlist[i].get("daySince") print(wlist[i].get("daySince")) prep = wlist[i].get("cloud") # print(prep) cut = jieba.cut(str(prep)) listcut = list(cut) # print(listcut) n_gramWords = _word_ngrams(tokens = listcut,ngram_range=(1,1)) preob["finalcloud"] = n_gramWords finalcloud.append(preob) # print(finalcloud) print('saving as json') with open('dawn.json', 'w', encoding="utf-8") as f: json.dump(finalcloud, f, indent=4, ensure_ascii=False) # for n_gramWord in n_gramWords: # print(n_gramWord) # 版权声明:本文为CSDN博主「姚贤贤」的原创文章,遵循 CC 4.0 BY-SA 版权协议,转载请附上原文出处链接及本声明。 # 原文链接:https://blog.csdn.net/u011311291/article/details/79269931
def write_data_export_zip(self, request, response): with zipfile.ZipFile(response, mode="w") as zip_file: with zip_file.open( f"{self.slug}.json", mode="w" ) as json_binary_file, io.TextIOWrapper(json_binary_file) as json_file: json.dump( { "email": self.user.email, "date_joined": self.user.date_joined.isoformat(), "last_login": self.user.last_login.isoformat(), "url": request.build_absolute_uri(self.get_absolute_url()), "is_public": self.is_public, "name": self.name, "city_or_town": self.city_or_town, "country": self.country, "cause_areas": list(map(CauseArea.label, self.cause_areas)), "cause_areas_other": self.cause_areas_other, "available_to_volunteer": self.available_to_volunteer, "open_to_job_offers": self.open_to_job_offers, "expertise_areas": list( map(ExpertiseArea.label, self.expertise_areas) ), "expertise_areas_other": self.expertise_areas_other, "career_interest_areas": list( map(ExpertiseArea.label, self.career_interest_areas) ), "available_as_speaker": self.available_as_speaker, "topics_i_speak_about": self.topics_i_speak_about, "organisational_affiliations": list( map( OrganisationalAffiliation.label, self.organisational_affiliations, ) ), "summary": self.summary, "giving_pledges": list( map(GivingPledge.label, self.giving_pledges) ), "member_of_local_groups": [ request.build_absolute_uri(local_group.get_absolute_uri()) for local_group in self.local_groups.all() ], "organiser_of_local_groups": [ request.build_absolute_uri(local_group.get_absolute_uri()) for local_group in self.user.localgroup_set.all() ], "aliases": [ request.build_absolute_uri( urls.reverse("profile", kwargs={"slug": slug.slug}) ) for slug in self.slugs.filter(redirect=True) ], "legacy_hub_url": ( self.legacy_record and request.build_absolute_uri( urls.reverse( "profile_legacy", kwargs={"legacy_record": self.legacy_record}, ) ) ), }, json_file, indent=2, ) if self.image: with self.image.open() as image_src_file, zip_file.open( self.slug + pathlib.PurePath(self.image.name).suffix, mode="w" ) as image_dst_file: shutil.copyfileobj(image_src_file, image_dst_file)
cv["cv%02d/all" % (cv_outer_i)] = [tr_val, te] cv_inner = StratifiedKFold(y[tr_val].ravel(), n_folds=NFOLDS_INNER, random_state=42) for cv_inner_i, (tr, val) in enumerate(cv_inner): cv["cv%02d/cvnested%02d" % ((cv_outer_i), cv_inner_i)] = [tr_val[tr], tr_val[val]] for k in cv: cv[k] = [cv[k][0].tolist(), cv[k][1].tolist()] C_range = [[100], [10], [1], [1e-1], [1e-2], [1e-3], [1e-4], [1e-5], [1e-6], [1e-7], [1e-8], [1e-9]] user_func_filename = "/home/ad247405/git/scripts/2016_schizConnect/supervised_analysis/all_studies+VIP/all_subjects/Freesurfer/ROIs/learning_curve_ratios/intra_site/svm_ratio_0_2.py" config = dict(data=dict(X="Xrois_vol+cov.npy", y="y.npy"), params=C_range, resample=cv, map_output="model_selectionCV", user_func=user_func_filename, reduce_input="results/*/*", reduce_group_by="params", reduce_output="model_selectionCV.csv") json.dump(config, open(os.path.join(WD, "config_dCV.json"), "w")) # Build utils files: sync (push/pull) and PBS import brainomics.cluster_gabriel as clust_utils sync_push_filename, sync_pull_filename, WD_CLUSTER = \ clust_utils.gabriel_make_sync_data_files(WD) cmd = "mapreduce.py --map %s/config_dCV.json" % WD_CLUSTER clust_utils.gabriel_make_qsub_job_files(WD, cmd)
def write_json(data: dict, dirname: str, subname: str): filepath = prepare_filepath(dirname, subname, JSON_EXT) LOG.debug(f"Writing JSON file: {filepath}") with open(filepath, "w") as fp: json.dump(data, fp, indent=2, sort_keys=True)
dbroot = 'jsondb/' for arg in sys.argv: if (argi > 0): print "// Debug: parsing " + arg + "\n" tree = ElementTree.parse(arg) root = tree.getroot() if root.tag != 'application': print "ERROR: XML root element is <%s> but expected <application>" % root.tag sys.exit(2) for objType in root.findall('objType'): typeName = objType.get('name') objects = [] fp = open(dbroot + typeName + '.json', 'w') if fp: print "Generating initial database for %s" % typeName id = 1 for obj in root.findall('object'): if (obj.get('type') == typeName): properties = dict(obj.attrib) properties['id'] = id for child in obj.findall('*'): properties[child.tag] = child.text objects.append(properties) id += 1 if len(objects) != 0: json.dump(objects, fp) fp.close() argi += 1
def write_min_json(data: dict, dirname: str, subname: str): filepath = prepare_filepath(dirname, subname, MIN_JSON_EXT) LOG.debug(f"Writing minified JSON file: {filepath}") with open(filepath, "w") as fp: json.dump(data, fp, separators=(",", ":"), sort_keys=True)
entry['cases'] += 1 entry_county = entry['counties'][p_county] entry_county['cases'] += 1 entry_county['age'][person[0]] += 1 else: skipped += 1 if unknown_in_fille > 0: print '# of unknown people:', unknown_in_fille t1 = time.clock() print '\t',n,'records in',(t1-t0),' secs' if len(unknown) > 0: print 'unknown: ',len(unknown) with open('unknown.txt', 'wb') as f: for item in unknown: print>>f, item print 'save json' t0 = time.clock() with open(os.path.join(dir_name, 'cases_per_day.json'), 'wb') as f: json.dump(cases_per_day, f, sort_keys=True) #indent=2, t1 = time.clock() print'\t', (t1-t0), ' secs' print 'done' # if skipped > 0: # print '\nskipped:', skipped
def save_json(fname, data): with open(fname, "w") as json_file: json.dump(data, json_file, indent=4, sort_keys=True)
def test( total_k_step, is_mc ): with tf.Graph().as_default(): loader = data_loader.data_loader( batch_size=BATCH_SIZE, dataset_path=PREPRO_DATASET_JSON, qa_data_path=PREPRO_QA_H5, img_tfrecord_size=IMG_TFRECORD_SIZE, max_words_q=MAX_WORDS_Q, dim_img=DIM_IMG, is_training=False ) num_test = loader.get_test_length() vocab_size = loader.get_vocabulary_size() answer_vocab_size = loader.get_answer_vocab_size() fname = 'eval_result/MultipleChoice_mscoco_val2014_dan_results.json' if is_mc else 'eval_result/OpenEnded_mscoco_val2014_dan_results.json' print 'total questions: ' + str(num_test) print 'vocabulary_size : ' + str(vocab_size) print 'file to write: ', fname with tf.Session() as sess: print 'constructing model...' model = DAN.Dual_Attention_Network( scope='dan', sess=sess, batch_size=BATCH_SIZE, k_step=total_k_step, dim_img=DIM_IMG, max_words_q=MAX_WORDS_Q, vocab_size=vocab_size, vocab_embed_size=VOCAB_EMBED_SIZE, rnn_size=RNN_SIZE, hidden_size=HIDDEN_SIZE, ans_num=answer_vocab_size, dan_npy_path=DAN_VAR_PATH) print 'model built' print 'initialize global variables..' sess.run(tf.global_variables_initializer()) tf.train.start_queue_runners(sess=sess) qa_dict = {} result = [] tStart_total = time.time() for itr in range(0, num_test, BATCH_SIZE): idx_batch, ques_id_batch, question_batch, ques_len_batch, img_idx_batch, img_feat_batch, MC_ans_batch = loader.get_next_test_batch(sess) predicted = model.predict_answers( img_feat_batch, question_batch, ques_len_batch, 1.0) loader.print_test_example( img_idx_batch[0], question_batch[0], predicted[0] ) avn1 = model.get_avn(1, img_feat_batch, question_batch, ques_len_batch, 1.0) avn2 = model.get_avn(2, img_feat_batch, question_batch, ques_len_batch, 1.0) avn_list = [avn1[0], avn2[0]] aut1 = model.get_aut(1, img_feat_batch, question_batch, ques_len_batch, 1.0) aut2 = model.get_aut(2, img_feat_batch, question_batch, ques_len_batch, 1.0) aut_list = [aut1[0], aut2[0]] loader.save_test_example( ques_id_batch[0], img_idx_batch[0], avn_list, question_batch[0], aut_list, predicted[0] ) if is_mc is False: # for open-ended top_ans = np.argmax(predicted, axis=1) else: # for multiple-choice for b in range(BATCH_SIZE): predicted[b, MC_ans_batch[b]] += 1 top_ans = np.argmax(predicted, axis=1) for i in range(0,BATCH_SIZE): ans = loader.get_answer( top_ans[i] ) if int(ques_id_batch[i]) not in qa_dict: qa_dict[int(ques_id_batch[i])] = ans if itr % 300*BATCH_SIZE ==0: print "Testing batch: ", itr print "Testing done." tStop_total = time.time() print "Total Time Cost:", round(tStop_total - tStart_total,2), "s" print 'Saving result...' for key in qa_dict.keys(): result.append({ u'question_id': key, u'answer': qa_dict.pop(key) }) print "total qa pairs: ", len(result) fname = 'eval_result/MultipleChoice_mscoco_val2014_dan_results.json' if is_mc else 'eval_result/OpenEnded_mscoco_val2014_dan_results.json' print 'wirte %s'% fname json.dump(result,open(fname,'w'))
ch += 4 * ' ' + '<title lang="ar">' + tit.replace( ''', "'").replace('"', '"').replace( '&', 'و'.decode('utf-8')).replace( '<div class="schedule__row__nowshowing">', '') + '</title>\n' ch += 4 * ' ' + '<desc lang="ar">' + de.replace( ''', "'").replace('"', '"').replace( '&', 'و'.decode('utf-8')).strip() + '</desc>\n </programme>\r' with io.open("/etc/epgimport/aljazeera.xml", "a", encoding="utf-8") as f: f.write(ch) print 'aljazeera epg download finished' else: 'No data found for aljazeera' with io.open("/etc/epgimport/aljazeera.xml", "a", encoding="utf-8") as f: f.write(('</tv>').decode('utf-8')) import json with open('/usr/lib/enigma2/python/Plugins/Extensions/Epg_Plugin/times.json', 'r') as f: data = json.load(f) for channel in data['bouquets']: if channel["bouquet"] == "aljazeera": channel['date'] = datetime.today().strftime('%A %d %B %Y at %I:%M %p') with open('/usr/lib/enigma2/python/Plugins/Extensions/Epg_Plugin/times.json', 'w') as f: json.dump(data, f)
def save_json(data, filename): import json with open(filename, 'w') as f: json.dump(data, f, indent=4)
def main(args): cfg = setup(args) logger = setup_logger(output=cfg.OUTPUT_DIR, distributed_rank=comm.get_rank(), name="imbalance detection") detector_ours = build_detector(cfg) gambler_ours = build_gambler(cfg) DetectionCheckpointer(detector_ours, save_dir=cfg.OUTPUT_DIR).resume_or_load( os.path.join(args.dir_ours, "model_0099999.pth"), resume=args.resume ) # DetectionCheckpointer(gambler_ours, save_dir=cfg.OUTPUT_DIR).resume_or_load( # os.path.join(args.dir_ours, "gambler_models/model_final.pth"), resume=args.resume # ) detector_baseline = build_detector(cfg) gambler_baseline = build_gambler(cfg) DetectionCheckpointer(detector_baseline, save_dir=cfg.OUTPUT_DIR).resume_or_load( os.path.join(args.dir_baseline, "model_final.pth"), resume=args.resume ) # DetectionCheckpointer(gambler_baseline, save_dir=cfg.OUTPUT_DIR).resume_or_load( # cfg.MODEL.GAMBLER_HEAD.WEIGHTS, resume=args.resume # ) dicts = list(DatasetCatalog.get(args.dataset)) metadata = MetadataCatalog.get(args.dataset) for idx, dataset_name in enumerate(cfg.DATASETS.TEST): test_data_loader = GANTrainer.build_test_loader(cfg, dataset_name) train_data_loader = GANTrainer.build_train_loader(cfg) evaluator_ours = GANTrainer.build_evaluator(cfg, dataset_name, os.path.join(args.dir_ours, "inference")) evaluator_baseline = GANTrainer.build_evaluator(cfg, dataset_name, os.path.join(args.dir_baseline, "inference")) analyzer = Analyzer(args.output) # visualize_inference(detector, gambler, train_data_loader, args.source) if args.per_image_ap: # import json # file_path = os.path.join("./output/gambler/2_full_coco/smoothing/fromscratch/focal_noimage/inference", # "coco_instances_results_mine.json") # with PathManager.open(file_path, "r") as f: # imgid_to_pred_ours = json.load(f) # # from collections import defaultdict # _imgid_to_pred = defaultdict(list) # for prediction in imgid_to_pred_ours: # _imgid_to_pred[prediction["image_id"]].append(prediction) # imgid_to_pred_ours = _imgid_to_pred # # file_path = os.path.join( # "./output/retinanet_baselines/simple_retinanet/simple_retina_fullcoco/all_layers_3scales/inference", # "coco_instances_results.json") # with PathManager.open(file_path, "r") as f: # imgid_to_pred_base = json.load(f) # # from collections import defaultdict # _imgid_to_pred_base = defaultdict(list) # for prediction in imgid_to_pred_base: # _imgid_to_pred_base[prediction["image_id"]].append(prediction) # imgid_to_pred_base = _imgid_to_pred_base # # # imgid_to_topk_APs={k: v for k, v in sorted(imgid_to_topk_APs.items(), key=lambda item: item[1])} # args.conf_threshold = 0.2 # a = dict(islice(imgid_to_topk_APs.items(), 200)) # # for dic in tqdm.tqdm(dicts): # if dic["image_id"] in a.keys(): # img = cv2.imread(dic["file_name"], cv2.IMREAD_COLOR)[:, :, ::-1] # basename = os.path.basename(dic["file_name"]) # predictions_ours = create_instances(imgid_to_pred_ours[dic["image_id"]], img.shape[:2]) # # predictions_ours.remove("pred_masks") # vis = Visualizer(img, metadata) # vis_pred = vis.draw_instance_predictions(predictions_ours).get_image() # # vis = Visualizer(img, metadata) # vis_gt = vis.draw_dataset_dict(dic).get_image() # # predictions_base = create_instances(imgid_to_pred_base[dic["image_id"]], img.shape[:2]) # # predictions_base.remove("pred_masks") # vis = Visualizer(img, metadata) # vis_pred_baseline = vis.draw_instance_predictions(predictions_base).get_image() # # concat = np.concatenate((vis_pred, vis_pred_baseline, vis_gt), axis=1) # cv2.imwrite(os.path.join(args.output, basename), concat[:, :, ::-1]) with EventStorage(0) as storage: analyzer.find_ap_per_img(detector_ours, test_data_loader, evaluator_ours) imgid_to_ap_ours, imgid_to_pred_ours = analyzer.imgid_to_AP, analyzer.imgid_to_pred analyzer.save() analyzer.find_ap_per_img(detector_baseline, test_data_loader, evaluator_baseline) imgid_to_ap_base, imgid_to_pred_base = analyzer.imgid_to_AP, analyzer.imgid_to_pred analyzer.save() imgid_to_topk_APs = get_topk_different_imgs(imgid_to_ap_ours, imgid_to_ap_base, k=args.k, order=args.sort_order) if args.sort_order == "desc": logger.info(f"The topk most different APs where our model outperformed") elif args.sort_order == "asc": logger.info(f"The topk most different APs where baseline outperformed") for dic in tqdm.tqdm(dicts): if dic["image_id"] in imgid_to_topk_APs.keys(): img = cv2.imread(dic["file_name"], cv2.IMREAD_COLOR)[:, :, ::-1] basename = os.path.basename(dic["file_name"]) predictions_ours = create_instances(imgid_to_pred_ours[dic["image_id"]], img.shape[:2]) predictions_ours.remove("pred_masks") vis = Visualizer(img, metadata) vis_pred = vis.draw_instance_predictions(predictions_ours).get_image() vis = Visualizer(img, metadata) vis_gt = vis.draw_dataset_dict(dic).get_image() predictions_base = create_instances(imgid_to_pred_base[dic["image_id"]], img.shape[:2]) predictions_base.remove("pred_masks") vis = Visualizer(img, metadata) vis_pred_baseline = vis.draw_instance_predictions(predictions_base).get_image() concat = np.concatenate((vis_pred, vis_pred_baseline, vis_gt), axis=1) cv2.imwrite(os.path.join(args.output, basename), concat[:, :, ::-1]) if args.compare_aps: with EventStorage(0) as storage: if os.path.isfile(os.path.join(args.dir_baseline, "results.json")): with PathManager.open(os.path.join(args.dir_baseline, "results.json"), "r") as f: results_baseline = json.load(f) else: results_baseline = load_old_inference_results(test_data_loader, evaluator_baseline) with PathManager.open(os.path.join(args.dir_baseline, "results.json"), "w") as f: json.dump(results_baseline, f, indent=2) if os.path.isfile(os.path.join(args.dir_ours, "results.json")): with PathManager.open(os.path.join(args.dir_ours, "results.json"), "r") as f: results_ours = json.load(f) else: results_ours = load_old_inference_results(test_data_loader, evaluator_ours) with PathManager.open(os.path.join(args.dir_ours, "results.json"), "w") as f: json.dump(results_ours, f, indent=2) plot_aps(cfg, args, results_ours, results_baseline, sort=args.sort) # results_ours_dict = OrderedDict() # results_baseline_dict = OrderedDict() # # results_ours_dict[dataset_name] = results_ours # results_baseline_dict[dataset_name] = results_baseline # # logger.info("Evaluation results for {} in csv format:".format(dataset_name)) # print_csv_format(results_ours_dict) # print_csv_format(results_baseline_dict) if args.with_gambler: with EventStorage(0) as storage: with torch.no_grad(): # with open(os.path.join(args.output, "statistics_gambler_fg.csv"), "a") as my_csv: # my_csv.write(f"img, " # f"the highest score predicted, " # f"gambler score before normalization, " # f"gambler score after normalization, " # f"Gt id, " # f"predicted class id, " # f"loss\n") # with open(os.path.join(args.output, "statistics_gambler_highloss.csv"), "a") as my_csv: # my_csv.write(f"img, " # f"the highest score predicted, " # f"gambler score before normalization, " # f"gambler score after normalization, " # f"Gt id, " # f"predicted class id, " # f"loss\n") # with open(os.path.join(args.output, "statistics_baseline_fg.csv"), "a") as my_csv: # my_csv.write(f"img, " # f"the highest score predicted, " # f"Gt id, " # f"predicted class id, " # f"loss\n") # with open(os.path.join(args.output, "statistics_baseline_highloss.csv"), "a") as my_csv: # my_csv.write(f"img, " # f"the highest score predicted, " # f"Gt id, " # f"predicted class id, " # f"loss\n") with torch.no_grad(): with open(os.path.join(args.output, f"{args.dir_ours.split('/')[-2]}_one_image_allanchors.csv"), "a") as my_csv: my_csv.write(f"img, " f"the highest score predicted, " f"gambler score before normalization, " f"gambler score after normalization, " f"Gt id, " f"predicted class id, " f"loss\n") for img_indx, inputs in enumerate(train_data_loader): # if img_indx > 2: # break input_images, generated_output, gt_classes, _, loss_dict = detector_ours(inputs) mask = torch.ones_like(gt_classes) gambler_loss_dict, weights, betting_map = gambler_ours(input_images, generated_output['pred_class_logits'], gt_classes, mask, detach_pred=False) # (N,AK,H,W) gt_classes = gt_classes.flatten() #torch.Size([37632]) pred_class_logits = list_N_AK_H_W_to_NsumHWA_K(generated_output['pred_class_logits'], cfg.MODEL.RETINANET.NUM_CLASSES) #torch.Size([37632, 80]) # pred_class_sigmoids = (torch.sigmoid(pred_class_logits) - 0.5) * 256 pred_class_sigmoids = torch.sigmoid(pred_class_logits) betting_map = list_N_AK_H_W_to_NsumHWA_K(betting_map, num_classes=1).flatten() # torch.Size([37632]) weights = weights.flatten() # torch.Size([37632]) loss_before_weighting = list_N_AK_H_W_to_NsumHWA_K(gambler_loss_dict['NAKHW_loss'], num_classes=1).flatten() # torch.Size([37632]) # foreground_idxs = (gt_classes >= 0) & (gt_classes != cfg.MODEL.RETINANET.NUM_CLASSES) # filtered_anchor_ind = foreground_idxs # print(f"number of filtered anchors to be displayed: {filtered_anchor_ind.sum()}") # # f_betting_map = betting_map[filtered_anchor_ind] # f_weights = weights[filtered_anchor_ind] # f_gt_classes = gt_classes[filtered_anchor_ind] # f_pred_class_sigmoids = pred_class_sigmoids[filtered_anchor_ind, :] # f_loss_before_weighting = loss_before_weighting[filtered_anchor_ind] # for over all chosen anchors for pred_class_sigmoid, gambler_score_bef, gambler_score_aft, matched_gt, loss in zip(pred_class_sigmoids, betting_map, weights, gt_classes, loss_before_weighting): with open(os.path.join(args.output, f"{args.dir_ours.split('/')[-2]}_one_image_allanchors.csv"), "a") as my_csv: my_csv.write(f"{inputs[0]['file_name'].split('/')[-1]}, " f"{pred_class_sigmoid.max()}, " f"{gambler_score_bef}, " f"{gambler_score_aft}, " f"{matched_gt}, " f"{pred_class_sigmoid.argmax()}, " f"{loss}\n") # foreground_idxs = (gt_classes >= 0) & (gt_classes != cfg.MODEL.RETINANET.NUM_CLASSES) # filtered_anchor_ind = (loss_before_weighting >= (0.1 * loss_before_weighting.max())) # print(f"number of filtered anchors to be displayed: {filtered_anchor_ind.sum()}") # # f_betting_map = betting_map[filtered_anchor_ind] # f_weights = weights[filtered_anchor_ind] # f_gt_classes = gt_classes[filtered_anchor_ind] # f_pred_class_sigmoids = pred_class_sigmoids[filtered_anchor_ind, :] # f_loss_before_weighting = loss_before_weighting[filtered_anchor_ind] # # # for over all chosen anchors # for pred_class_sigmoid, gambler_score_bef, gambler_score_aft, matched_gt, loss in zip( # f_pred_class_sigmoids, f_betting_map, f_weights, f_gt_classes, f_loss_before_weighting): # with open(os.path.join(args.output, "statistics_gambler_highloss.csv"), "a") as my_csv: # my_csv.write(f"{inputs[0]['file_name'].split('/')[-1]}, " # f"{pred_class_sigmoid.max()}, " # f"{gambler_score_bef}, " # f"{gambler_score_aft}, " # f"{matched_gt}, " # f"{pred_class_sigmoid.argmax()}, " # f"{loss}\n") ############################################################################################## # input_images, generated_output, gt_classes, loss_dict = detector_baseline(inputs) # # gt_classes = gt_classes.flatten() # torch.Size([37632]) # pred_class_logits = list_N_AK_H_W_to_NsumHWA_K(generated_output['pred_class_logits'], # cfg.MODEL.RETINANET.NUM_CLASSES) # torch.Size([37632, 80]) # # pred_class_sigmoids = (torch.sigmoid(pred_class_logits) - 0.5) * 256 # pred_class_sigmoids = torch.sigmoid(pred_class_logits) # # loss_before_weighting = list_N_AK_H_W_to_NsumHWA_K(gambler_loss_dict['NAKHW_loss'], # # num_classes=1).flatten() # torch.Size([37632]) # # foreground_idxs = (gt_classes >= 0) & (gt_classes != cfg.MODEL.RETINANET.NUM_CLASSES) # filtered_anchor_ind = foreground_idxs # print(f"number of filtered anchors to be displayed: {filtered_anchor_ind.sum()}") # # # f_betting_map = betting_map[filtered_anchor_ind] # # f_weights = weights[filtered_anchor_ind] # f_gt_classes = gt_classes[filtered_anchor_ind] # f_pred_class_sigmoids = pred_class_sigmoids[filtered_anchor_ind, :] # # f_loss_before_weighting = loss_before_weighting[filtered_anchor_ind] # # # for over all chosen anchors # for pred_class_sigmoid, matched_gt in zip(f_pred_class_sigmoids, f_gt_classes): # with open(os.path.join(args.output, "statistics_baseline_fg.csv"), "a") as my_csv: # my_csv.write(f"{inputs[0]['file_name'].split('/')[-1]}, " # f"{pred_class_sigmoid.max()}, " # f"{matched_gt}, " # f"{pred_class_sigmoid.argmax()}\n") # # foreground_idxs = (gt_classes >= 0) & (gt_classes != cfg.MODEL.RETINANET.NUM_CLASSES) # filtered_anchor_ind = (loss_before_weighting >= (0.1 * loss_before_weighting.max())) # print(f"number of filtered anchors to be displayed: {filtered_anchor_ind.sum()}") # # f_betting_map = betting_map[filtered_anchor_ind] # f_weights = weights[filtered_anchor_ind] # f_gt_classes = gt_classes[filtered_anchor_ind] # f_pred_class_sigmoids = pred_class_sigmoids[filtered_anchor_ind, :] # f_loss_before_weighting = loss_before_weighting[filtered_anchor_ind] # # # for over all chosen anchors # for pred_class_sigmoid, gambler_score_bef, gambler_score_aft, matched_gt, loss in zip( # f_pred_class_sigmoids, f_betting_map, f_weights, f_gt_classes, f_loss_before_weighting): # with open(os.path.join(args.output, "statistics_gambler_highloss.csv"), "a") as my_csv: # my_csv.write(f"{inputs[0]['file_name'].split('/')[-1]}, " # f"{pred_class_sigmoid.max()}, " # f"{gambler_score_bef}, " # f"{gambler_score_aft}, " # f"{matched_gt}, " # f"{pred_class_sigmoid.argmax()}, " # f"{loss}\n") break if args.vis_bets: from train_net import visualize_training_ with torch.no_grad(): with EventStorage(0) as storage: for img_indx, inputs in enumerate(train_data_loader): # if img_indx > 2: # break input_images, generated_output, gt_classes, _, loss_dict = detector_ours(inputs) mask = torch.ones_like(gt_classes) gambler_loss_dict, weights, betting_map = gambler_ours(input_images, generated_output['pred_class_logits'], gt_classes, mask, detach_pred=False) # (N,AK,H,W) visualize_training_(gt_classes.clone().detach(), gambler_loss_dict["NAKHW_loss"], weights, input_images, storage)
for _max in dmax: if _max not in resultado[_type].keys(): resultado[_type][_max] = {'acerto':0,'erro':0} _distances = [m.distance for m in match if m.distance < _max] #print get_pos(t),get_pos(d), d, t print len(_distances), check_hit(get_pos(t), get_pos(d)), get_pos(t), get_pos(d) if len(_distances)> 5: if check_hit(get_pos(t), get_pos(d)): resultado[_type][_max]['acerto'] +=1 else: resultado[_type][_max]['erro'] +=1 with open('json_data/resultado_01.json', 'w') as fp: json.dump(resultado, fp) #print resultado # sys.exit() #**********************************************************************************************
def save_to_json(fname, data): with open(fname, 'w') as outfile: json.dump(data, outfile)
def save(self): with open(Config.path, 'w') as f: json.dump(self, f, default=Config.config_to_dict)
async def inline_answer(_, inline_query: InlineQuery): results = [] i_q = inline_query.query string = i_q.lower() str_x = i_q.split(" ", 2) string_split = string.split() if inline_query.from_user and inline_query.from_user.id == Config.OWNER_ID or inline_query.from_user.id in Config.SUDO_USERS: MAIN_MENU = InlineQueryResultArticle( title="Main Menu", input_message_content=InputTextMessageContent( " 𝐔𝐒𝐄𝐑𝐆𝐄-𝐗 𝗠𝗔𝗜𝗡 𝗠𝗘𝗡𝗨 "), url="https://github.com/code-rgb/USERGE-X", description="Userge-X Main Menu", thumb_url="https://i.imgur.com/1xsOo9o.png", reply_markup=InlineKeyboardMarkup(main_menu_buttons())) results.append(MAIN_MENU) if string == "syntax": owner = [[ InlineKeyboardButton(text="Contact", url="https://t.me/deleteduser420") ]] results.append( InlineQueryResultPhoto( photo_url= "https://coverfiles.alphacoders.com/123/123388.png", caption="Hey I solved **𝚂𝚢𝚗𝚝𝚊𝚡's ░ Σrr♢r**", reply_markup=InlineKeyboardMarkup(owner))) if string == "rick": rick = [[ InlineKeyboardButton( text="🔍", url="https://www.youtube.com/watch?v=dQw4w9WgXcQ") ]] results.append( InlineQueryResultArticle( title="Not a Rick Roll", input_message_content=InputTextMessageContent( "Search Results"), description="Definately Not a Rick Roll", thumb_url="https://i.imgur.com/hRCaKAy.png", reply_markup=InlineKeyboardMarkup(rick))) if string == "alive": random_alive = random.choice(ALIVE_IMGS) buttons = [[ InlineKeyboardButton("🔧 SETTINGS", callback_data="settings_btn"), InlineKeyboardButton(text="⚡️ REPO", url=Config.UPSTREAM_REPO) ]] alive_info = f""" **[USERGE-X](https://github.com/code-rgb/USERGE-X) is Up and Running** • 🐍 Python : `v{versions.__python_version__}` • 🔥 Pyrogram : `v{versions.__pyro_version__}` • 🧬 𝑿 : `v{get_version()}` 🕔 Uptime : {userge.uptime} """ results.append( InlineQueryResultPhoto( photo_url=random_alive, caption=alive_info, reply_markup=InlineKeyboardMarkup(buttons))) if string == "geass": results.append( InlineQueryResultAnimation( animation_url="https://i.imgur.com/DeZHcRK.gif", caption="To defeat evil, I must become a greater evil", )) if string == "gapps": buttons = [[ InlineKeyboardButton("Open GApps", callback_data="open_gapps"), InlineKeyboardButton("Flame GApps", callback_data="flame_gapps") ], [ InlineKeyboardButton("Nik GApps", callback_data="nik_gapps") ]] results.append( InlineQueryResultArticle( title="GApps", input_message_content=InputTextMessageContent( "[\u200c](https://i.imgur.com/BZBMrfn.jpg) **LATEST Android 10 arm64 GApps**" ), description= "Get Latest GApps Download Links Directly from SF", thumb_url="https://i.imgur.com/Npzw8Ph.png", reply_markup=InlineKeyboardMarkup(buttons))) if len(string_split) == 2: #workaround for list index out of range if string_split[0] == "ofox": codename = string_split[1] t = TelegraphPoster(use_api=True) t.create_api_token('Userge-X') photo = "https://i.imgur.com/582uaSk.png" api_host = 'https://api.orangefox.download/v2/device/' try: cn = requests.get(f"{api_host}{codename}") r = cn.json() except ValueError: return s = requests.get( f"{api_host}{codename}/releases/stable/last").json() info = f"📱 **Device**: {r['fullname']}\n" info += f"👤 **Maintainer**: {r['maintainer']['name']}\n\n" recovery = f"🦊 <code>{s['file_name']}</code>\n" recovery += f"📅 {s['date']}\n" recovery += f"ℹ️ **Version:** {s['version']}\n" recovery += f"📌 **Build Type:** {s['build_type']}\n" recovery += f"🔰 **Size:** {s['size_human']}\n\n" recovery += "📍 **Changelog:**\n" recovery += f"<code>{s['changelog']}</code>\n\n" msg = info msg += recovery notes_ = s.get('notes') if notes_: notes = t.post(title='READ Notes', author="", text=notes_) buttons = [[ InlineKeyboardButton("🗒️ NOTES", url=notes['url']), InlineKeyboardButton("⬇️ DOWNLOAD", url=s['url']) ]] else: buttons = [[ InlineKeyboardButton(text="⬇️ DOWNLOAD", url=s['url']) ]] results.append( InlineQueryResultPhoto( photo_url=photo, thumb_url="https://i.imgur.com/o0onLYB.jpg", title="Latest OFOX RECOVERY", description=f"For device : {codename}", caption=msg, reply_markup=InlineKeyboardMarkup(buttons))) if string == "repo": results.append(REPO_X) if str_x[0].lower() == "op" and len(str_x) > 1: txt = i_q[3:] # TODO change it opinion = os.path.join(PATH, "emoji_data.txt") try: view_data = json.load(open(opinion)) except: view_data = False if view_data: # Uniquely identifies an inline message new_id = {int(inline_query.id): [{}]} view_data.update(new_id) json.dump(view_data, open(opinion, 'w')) else: d = {int(inline_query.id): [{}]} json.dump(d, open(opinion, 'w')) buttons = [[ InlineKeyboardButton( "👍", callback_data=f"op_y_{inline_query.id}"), InlineKeyboardButton( "👎", callback_data=f"op_n_{inline_query.id}") ]] results.append( InlineQueryResultArticle( title="Ask For Opinion", input_message_content=InputTextMessageContent(txt), description="e.g @yourbot op Are Cats Cute?", thumb_url="https://i.imgur.com/Zlc98qS.jpg", reply_markup=InlineKeyboardMarkup(buttons))) if string == "buttonnn": async for data in BUTTON_BASE.find(): button_data = data['msg_data'] text, buttons = pb(button_data) try: photo_url = data['photo_url'] except KeyError: photo_url = None if photo_url: results.append( InlineQueryResultPhoto(photo_url=photo_url, caption=text, reply_markup=buttons)) else: results.append( InlineQueryResultArticle( title=text, input_message_content=InputTextMessageContent( text), reply_markup=buttons)) if str_x[0].lower() == "secret": if len(str_x) == 3: user_name = str_x[1] msg = str_x[2] try: a = await userge.get_users(user_name) user_id = a.id except: return secret = os.path.join(PATH, "secret.txt") try: view_data = json.load(open(secret)) except: view_data = False if view_data: # Uniquely identifies an inline message new_id = { str(inline_query.id): { 'user_id': user_id, 'msg': msg } } view_data.update(new_id) json.dump(view_data, open(secret, 'w')) else: d = { str(inline_query.id): { 'user_id': user_id, 'msg': msg } } json.dump(d, open(secret, 'w')) buttons = [[ InlineKeyboardButton( "🔐 REVEAL", callback_data=f"secret_{inline_query.id}") ]] results.append( InlineQueryResultArticle( title="Send A Secret Message", input_message_content=InputTextMessageContent( f"📩 <b>TOPSECRET!</b> for {user_name}. Only he/she can open it." ), description=f"Send Secret Message to: {user_name}", thumb_url="https://i.imgur.com/c5pZebC.png", reply_markup=InlineKeyboardMarkup(buttons))) else: results = [(InlineQueryResultArticle( title="Send A Secret Message", input_message_content=InputTextMessageContent( "Do `.secret` for more info"), description="secret @username message ..."))] await inline_query.answer(results=results, cache_time=1, switch_pm_text="🔒 SECRETS", switch_pm_parameter="start") return else: results.append(REPO_X) try: if not len(results) == 0: await inline_query.answer(results=results, cache_time=1) except MessageEmpty: return
def persist_map(self): with open(self.map_file, 'w') as fh: json.dump(self.map, fh)
async def decode_records(records): total = len(records) print("Fetching details") async with aiohttp.ClientSession() as session: for i, r in enumerate(records): print("({}/{})Processing {}".format(i + 1, total, r)) path = os.path.join(basedir, r) with open(path) as f: data = json.load(f) if "data" in data: print("({}/{})Data present in {}, skipping".format(i + 1, total, r)) continue if "dataUrl" in data: url = data["dataUrl"] print("({}/{})Fetching details for {}".format(i + 1, total, r)) async with session.get(url) as res: details = await res.read() print("({}/{})Fetched details for {}".format(i + 1, total, r)) data["data"] = base64.b64encode(details).decode() with open(path, "w") as f: print("({}/{})Saving {}".format(i + 1, total, r)) json.dump(data, f, indent=2, ensure_ascii=False) continue print("({}/{})Neither data or dataUrl in {}, skipping".format(i + 1, total, r)) print("Decoding details") for i, r in enumerate(records): print("({}/{})Processing {}".format(i + 1, total, r)) path = os.path.join(basedir, r) with open(path) as f: data = json.load(f) if "details" in data: print("({}/{})Details present in {}, skipping".format(i + 1, total, r)) continue blob = base64.b64decode(data['data']) wrapper = pb.Wrapper() wrapper.ParseFromString(blob) assert wrapper.name == '.lq.GameDetailRecords' records_data = wrapper.data records = pb.GameDetailRecords() records.ParseFromString(records_data) records = records.records results = [] for record in records: wrapper = pb.Wrapper() wrapper.ParseFromString(record) name = wrapper.name.replace(".lq.", "") cls = getattr(pb, name) obj = cls() obj.ParseFromString(wrapper.data) result = MessageToDict(obj) result['@type'] = name results.append(result) data["details"] = results with open(path, "w") as f: print("({}/{})Saving {}".format(i + 1, total, r)) json.dump(data, f, indent=2, ensure_ascii=False)
def saveConfig(self): with open('config.json', 'w') as fp: json.dump(self.config, fp, sort_keys=True, indent=4)
# if the key matches the hashtag print that part of the data for hashTag in v: # the word variable is processed as an entire string not a list words = v[hashTag] # taking the [] placheholders out as in this they do not signify a list type words = words.replace('[','') words = words.replace(']','') hashtags.append([words]) if i == "date": for date in v: # find the current date current_date = v[date] dates.append(current_date) # combine both of these list into one combinedList = list(zip(dates,hashtags)) # sort that list by most recent date first combinedList = sorted(combinedList, key = lambda x:x[0], reverse=True) #Include the list into dictionary data["HashtagsByDate"] = combinedList #Append the updated column into original JSON file with open('output.json','a') as file_df: data = json.dump(data,file_df) file_df.close()
def next_idiom_list(strx): """ string to int list """ ret_list = list() strxs = strx.split() for i in range(len(data)): stri = data[i]["english"] stris = stri.split() if strxs[-1] == stris[0]: ret_list.append(i) return ret_list new_list = list() for i in range(n): if (i % 1000) == 0: print(i, "idioms finished.") idiom = data[i] new_data = dict() new_data["next"] = next_idiom_list(idiom["english"]) new_data["next_num"] = len(new_data["next"]) new_data["word"] = idiom["word"] new_data["english"] = idiom["english"] new_data["no"] = i new_list.append(new_data) # sorting sorted_d = sorted(new_list, key=lambda x: x['next_num']) with open('../data/idiom-graph.json', 'w+') as outfile: json.dump(sorted_d, outfile)
def save_json(data, path): with open(path, 'w') as f: json.dump(data, f)
def clear(): with open("./json/task.json", 'w') as updatejson: json.dump([], updatejson) return {"Success": "Successfully Deleted"}, 200, {'Content-type': 'application/json'}
from jina.peapods.runtimes.asyncio.rest.app import get_fastapi_app JINA_LOGO_URL = 'https://api.jina.ai/logo/logo-product/jina-core/horizontal-layout/colored/Product%20logo_Core_vertical_colorful%402x-margin.png' DAEMON_SCHEMA_FILENAME = 'daemon.json' GATEWAY_SCHEMA_FILENAME = 'gateway.json' args = set_gateway_parser().parse_args([]) logger = JinaLogger('') gateway_app = get_fastapi_app(args, logger) gateway_schema = gateway_app.openapi() gateway_schema['info']['x-logo'] = {'url': JINA_LOGO_URL} gateway_schema['servers'] = [] gateway_schema['servers'].append({ 'url': f'http://localhost:{args.port_expose}', 'description': 'Local Jina gateway' }) with open(GATEWAY_SCHEMA_FILENAME, 'w') as f: json.dump(gateway_schema, f) args = get_main_parser().parse_args([]) daemon_app = _get_app() daemon_schema = daemon_app.openapi() daemon_schema['info']['x-logo'] = {'url': JINA_LOGO_URL} daemon_schema['servers'] = [] daemon_schema['servers'].append({ 'url': f'http://localhost:{args.port_expose}', 'description': 'Local JinaD server' }) with open(DAEMON_SCHEMA_FILENAME, 'w') as f: json.dump(daemon_schema, f)
from collections import namedtuple import ast import json from pathlib import Path for p in Path('nb201/').glob("*.log"): archs = [] with open(p) as f: for line in f: if "Anytime results: " in line: arch_eval = ast.literal_eval(line[line.index("{'cifar10-valid':"):].rstrip()) archs.append(arch_eval) valid_acc = [] with open(p) as f: for line in f: if "Validation accuracy:" in line: pos = line.index("Validation accuracy: ") val = ast.literal_eval(line[pos + 21:pos + 29]) valid_acc.append(val) with open('nb201/{}.json'.format(p.stem), 'w') as f: json.dump({ 'test_acc': archs, 'valid_acc': valid_acc}, f)
r = requests.get(url.format(**var)) j = json.loads(r.content.decode("utf-8")) yield j['results'] movies = list(crawl_movies(total_pages)) movies = list(itertools.chain.from_iterable(movies)) return movies movies = crawl_top_rated_movies_id() # Save top_rated movies data_file = os.path.join(data_folder, 'top_rated_movies.json') with open(data_file, 'w') as stream: json.dump(movies, stream) # Load top_rated movies and transform to dictionary data_file = os.path.join(data_folder, 'top_rated_movies.json') with open(data_file, 'r') as stream: movies_load = json.load(stream) movies_load = dict([(d['id'], d) for d in movies_load]) def crawl_movies_detail(movie_id): var = { 'movie_id': movie_id, 'api_key': config['api_key'], 'language': 'en-US', }