def ana_results_save(self):
self.table.selectAll()
items = self.table.selectedItems()
deleted_keywords = {}
modified_keywords = {}
new_keywords = {}
# TODO gérer la modification du texte
for item in items:
item_row = item.row()
item_column = item.column()
equiv_key = "(" + str(item_row) + "," + str(item_column) + ")"
key = self.equivalence_table[equiv_key]
if item.checkState() == Qt.Unchecked:
# Mots-clés décochés
deleted_keywords[key] = {"modification": "supprime"}
elif item.checkState() == Qt.Checked and item.text() != key:
# Mots-cles modifiés
modified_keywords[key] = {"modification": "modifie", "nouveau_mot_cle": item.text()}
added_keywords = self.ana_results_add_keyword.text().split(";")
for new_keyword in added_keywords:
new_keywords[new_keyword.strip()] = {"modification": "ajoute"}
modified_keywords.update(deleted_keywords)
modified_keywords.update(new_keywords)
with open(self.project_directory + "/output/modified_keywords.json", "w", encoding="utf8") as outfile:
json.dump(modified_keywords, outfile, ensure_ascii=False, indent=4)
outfile.close()
self.ana_results_saved.setText("Résultats sauvegardés")
def export_json():
print('Exporting json')
# organization json file
bulk = []
entries = Organization.objects.all()
for e in entries:
x = {'Organization': e.name}
bulk.append(x)
with io.open('./cos/static/toporgs.json', 'w', encoding='utf8') as \
data_file:
json.dump(bulk, data_file, indent=1, ensure_ascii=False)
print('Finished exporting toporgs.json')
# journal first json file
save_to_journal_json('rrjournals', 'registered', 1)
# second json file
save_to_journal_json('rrjournalssome', 'featured', 1)
# third json file
save_to_journal_json('rrjournalsspecial', 'special', 1)
# fourth json file
save_to_journal_json('preregjournals', 'preregistered', 2)
# fifth json file
save_to_journal_json('topjournals', 'top', 2)
print('Finished exporting journal json files')
print("Completed exporting json")
def p4():
cur.execute("SELECT year, Sex, Marital_Status, Age_Value FROM mortality")
i = 0
while True:
try:
data = cur.fetchone()
i+=1
if i%500000 == 0:
print i
if int(data[0]) not in YSMA.keys():
YSMA[int(data[0])] = {str(data[1]): {str(data[2]): [int(data[3])]}}
elif str(data[1]) not in YSMA[int(data[0])]:
YSMA[int(data[0])][str(data[1])] = {str(data[2]): [int(data[3])]}
elif str(data[2]) not in YSMA[int(data[0])][str(data[1])]:
YSMA[int(data[0])][str(data[1])][str(data[2])] = [int(data[3])]
else:
YSMA[int(data[0])][str(data[1])][str(data[2])].append(int(data[3]))
except:
break
with open('C:\Users\jmorris\Documents\Fall_2015\Data Visualization\HW4/YearSexMarriageAge.json', 'w') as outfile: #Julian's file path
json.dump(YSMA, outfile, indent = 4)
return YSMA
def save_settings_export(properties):
settings = {
"option_export_scene" : properties.option_export_scene,
"option_embed_meshes" : properties.option_embed_meshes,
"option_url_base_html" : properties.option_url_base_html,
"option_copy_textures" : properties.option_copy_textures,
"option_lights" : properties.option_lights,
"option_cameras" : properties.option_cameras,
"option_animation" : properties.option_animation,
"option_frame_step" : properties.option_frame_step,
"option_all_meshes" : properties.option_all_meshes,
"option_flip_yz" : properties.option_flip_yz,
"option_materials" : properties.option_materials,
"option_normals" : properties.option_normals,
"option_colors" : properties.option_colors,
"option_uv_coords" : properties.option_uv_coords,
"option_edges" : properties.option_edges,
"option_faces" : properties.option_faces,
"option_vertices" : properties.option_vertices,
"option_vertices_truncate" : properties.option_vertices_truncate,
"option_scale" : properties.option_scale,
"align_model" : properties.align_model
}
fname = get_settings_fullpath()
f = open(fname, "w")
json.dump(settings, f)
def copy_json(input_path, output_path):
with open(input_path) as input:
with open(output_path, "w+") as output:
json.dump(
json.load(input),
output,
indent=2)
def body(self):
if self.playback:
subscription_id = MOCKED_SUBSCRIPTION_ID
else:
subscription_id = self.cmd('account list --query "[?isDefault].id" -o tsv')
role_name = 'cli-test-role'
template = {
"Name": "Contoso On-call",
"Description": "Can monitor compute, network and storage, and restart virtual machines",
"Actions": ["Microsoft.Compute/*/read",
"Microsoft.Compute/virtualMachines/start/action",
"Microsoft.Compute/virtualMachines/restart/action",
"Microsoft.Network/*/read",
"Microsoft.Storage/*/read",
"Microsoft.Authorization/*/read",
"Microsoft.Resources/subscriptions/resourceGroups/read",
"Microsoft.Resources/subscriptions/resourceGroups/resources/read",
"Microsoft.Insights/alertRules/*",
"Microsoft.Support/*"],
"AssignableScopes": ["/subscriptions/{}".format(subscription_id)]
}
template['Name'] = role_name
_, temp_file = tempfile.mkstemp()
with open(temp_file, 'w') as f:
json.dump(template, f)
role = self.cmd('role create --role-definition {}'.format(temp_file.replace('\\', '\\\\')), None)
self.cmd('role list -n {}'.format(role_name), checks=[JMESPathCheck('[0].properties.roleName', role_name)])
self.cmd('role delete -n {}'.format(role_name), None)
self.cmd('role list -n {}'.format(role_name), NoneCheck())
def main():
api_url, username, password = get_account_data(True)
backup_files = get_nsbackup_files(True)
if 'pickle_backup_file' in backup_files.keys():
from pickle import Pickler
if 'json_backup_file' in backup_files.keys():
import json
# Instantiate the inwx class (does not connect yet but dispatches calls to domrobot objects with the correct API URL
inwx_conn = domrobot(api_url, username, password, 'en', False)
# get the list of all domains:
domains = inwx_conn.nameserver.list()['domains']
# get all the nameserver entries for each domain
current, total = 0, len(domains)
nsentries = dict()
for domain in domains:
current += 1
domain = domain['domain']
print "%i of %i - Currently backing up %s." % (current, total, domain)
nsentries[domain] = inwx_conn.nameserver.info({'domain': domain})['record']
if 'pickle_backup_file' in backup_files.keys():
Pickler(open(backup_files['pickle_backup_file'],'wb')).dump(nsentries)
print "Wrote backup file using Python Module Pickle : %s." % backup_files['pickle_backup_file']
if 'json_backup_file' in backup_files.keys():
json.dump(nsentries, open(backup_files['json_backup_file'], 'w'))
print "Wrote backup file using Python Module JSON: %s." % backup_files['json_backup_file']
def write(self, rootdir):
"""
Write the metadata to a JSON file in the rootdir.
Values contained in the metadata are:
first_trading_day : string
'YYYY-MM-DD' formatted representation of the first trading day
available in the dataset.
minute_index : list of integers
nanosecond integer representation of the minutes, the enumeration
of which corresponds to the values in each bcolz carray.
ohlc_ratio : int
The factor by which the pricing data is multiplied so that the
float data can be stored as an integer.
"""
metadata = {
'first_trading_day': str(self.first_trading_day.date()),
'minute_index': self.minute_index.asi8.tolist(),
'market_opens': self.market_opens.values.
astype('datetime64[m]').
astype(int).tolist(),
'market_closes': self.market_closes.values.
astype('datetime64[m]').
astype(int).tolist(),
'ohlc_ratio': self.ohlc_ratio,
}
with open(self.metadata_path(rootdir), 'w+') as fp:
json.dump(metadata, fp)
def auth_dialog(self):
try:
with open(CONF_FILE) as conf:
token = json.load(conf)
except (OSError, json.JSONDecodeError, KeyError):
print("Your token file doesn't exist or is malformed.")
print("If you WANT to proceed anonymously, pass"
"`-a/--anonymous` on the command line.")
print("Or you can create a new token now. "
"Create token? [y/n] ", end='')
if input().lower() in ('y', 'yes'):
username = input('Username: '******'gist3_{}'.format(id_gen())
resp = req.post(API_BASE + '/authorizations',
auth=(username, password),
json={'scopes': ['gist'],
'note': note})
if resp.status_code == req.codes.created:
token = resp.json()['token']
with open(CONF_FILE, 'w') as conf:
json.dump(token, conf)
print('Token created & saved to {}'.format(CONF_FILE))
else:
print('There was an error from github: ')
print(json.dumps(resp.json(), sort_keys=True, indent=4))
sys.exit(2)
else:
print('Aborting...')
sys.exit(0)
return AccessTokenAuth(token)
def delete(self, thema, id, beitragID=None):
discussionpath = "./data/themen/" + thema + "/" + id + ".json"
with open(discussionpath, "r") as discussionfile:
discussion = json.load(discussionfile)
if beitragID == None:
if discussion["Status"] == "deleted":
discussion["Status"] = " "
else:
discussion["Status"] = "deleted"
discussion["Bearbeiter"] = cherrypy.session["Benutzername"]
discussion["Bearbeitet"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
else:
for post in discussion["Beitraege"]:
if post["ID"] == beitragID:
if post["Status"] == "deleted":
post["Status"] = " "
else:
post["Status"] = "deleted"
post["Bearbeiter"] = cherrypy.session["Benutzername"]
post["Bearbeitet"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
with open(discussionpath, "w") as discussionfile:
json.dump(discussion, discussionfile, indent=4)
def get_opendata2_courses():
good_courses = 0
file_name = os.path.join(os.path.realpath(os.path.dirname(__file__)),
'%s/opendata2_departments.json' % c.DEPARTMENTS_DATA_DIR)
with open(file_name) as departments_file:
departments = json.load(departments_file)
# Create a text file for every department
for d in departments:
department = d['subject']
open_data_json = requests.get(
'https://api.uwaterloo.ca/v2/courses/{0}.json?key={1}'.format(
department.upper(), s.OPEN_DATA_API_KEY)).json
open_data_catalog_numbers = []
for course in open_data_json['data']:
open_data_catalog_numbers.append(course['catalog_number'])
# We now poll the individual endpoints of each course for the data
current_dep_json = []
course_url = 'https://api.uwaterloo.ca/v2/courses/{0}/{1}.json?key={2}'
for course in open_data_catalog_numbers:
good_courses += 1
json_data = requests.get(course_url.format(department.upper(),
course, s.OPEN_DATA_API_KEY)).json
current_dep_json.append(json_data['data'])
out_file_name = os.path.join(
os.path.realpath(os.path.dirname(__file__)),
'opendata2_courses/%s.json' % department.lower())
with open(out_file_name, 'w') as courses_out:
json.dump(current_dep_json, courses_out)
print 'Found {num} good courses'.format(num=good_courses)
def main():
# THIS CODE IS RUN ONE TIME ONLY (#SAVETHESERVERS)
# For Champions:
#api = RiotAPI('3e888957-13a3-4ba2-901c-fae3e421d998')
#r = api.get_championList()
#with open('testChampion.json', 'w') as outfile:
# json.dump(r, outfile, indent=4)
# --------------------------------------------------------
with open('testChampion.json', 'r') as data_file:
data = json.load(data_file)
championIds = [data['data'][championName]['id']
for championName in data['data']]
championNames = [championName for championName in data['data']]
championTitles = [data['data'][championName]['title']
for championName in data['data']]
championReference = dict(zip(championIds,
zip(championNames, championTitles)))
with open('championReference.json', 'w') as outfile:
json.dump(championReference, outfile, indent=4)
def do_deployment_manifest(system, configuration):
app.log(system, "Creating deployment manifest in", system['sandbox'])
data = {'configuration': configuration}
metafile = os.path.join(system['sandbox'], 'baserock', 'deployment.meta')
with app.chdir(system['sandbox']), open(metafile, "w") as f:
json.dump(data, f, indent=4, sort_keys=True, encoding='unicode-escape')
f.flush()
def _update_manifest(self,dom_id,ts_esmf,var,kv):
"""
Adds a key-value set to the dictionary storing metadata for time ts_esmf and variable var.
:param dom_id: the domain id (1, 2, 3, ...)
:param ts_esmf: ESMF time string
:param var: variable name
:param kv: key-value dictionary to merge
"""
# update the manifest with the domain/ts_esmf/var info
dom = self.manifest.get(str(dom_id), {})
self.manifest[str(dom_id)] = dom
# extract timestamp
td = dom.get(ts_esmf, {})
dom[ts_esmf] = td
# store variable in timestamp
vd = td.get(var, {})
td[var] = vd
vd.update(kv)
mf_path = os.path.join(self.output_path, self.product_name + '.json')
# synchronize the file
json.dump(self.manifest, open(mf_path, 'w'))
def dump_app(id, **kw):
# Because @robhudson told me to.
from mkt.api.resources import AppResource
# Note: not using storage because all these operations should be local.
target_dir = os.path.join(settings.DUMPED_APPS_PATH, 'apps',
str(id / 1000))
target_file = os.path.join(target_dir, str(id) + '.json')
try:
obj = Webapp.objects.get(pk=id)
except Webapp.DoesNotExist:
task_log.info(u'Webapp does not exist: {0}'.format(id))
return
req = RequestFactory().get('/')
req.user = AnonymousUser()
req.REGION = WORLDWIDE
if not os.path.exists(target_dir):
os.makedirs(target_dir)
task_log.info('Dumping app {0} to {1}'.format(id, target_file))
res = AppResource().dehydrate_objects([obj], request=req)
json.dump(res[0], open(target_file, 'w'), cls=JSONEncoder)
return target_file
def saveTweets(self):
meaningful = self.jsonAccepted*self.cfg['KeepAccepted'] + self.jsonPartial*self.cfg['KeepPartial'] + self.jsonExcluded*self.cfg['KeepExcluded']
if len(meaningful)>1:
print "\nDumping tweets to file, contains %s tweets with %s accepted, %s rejected, %s partial matches, and %s irrelevant" % (len(meaningful),
self.acceptedCount,
self.excludedCount,
self.partialCount,
self.irrelevantCount)
if self.cfg['TweetData'] != 'all':
meaningful = cleanJson(meaningful,self.cfg,self.tweetTypes)
#timeStamp = datetime.date.today().strftime("%A")
timeStamp = self.startTime
self.lastWrite = self.startDay
if self.cfg['KeepRaw']:
with open(self.pathOut+'Raw_'+self.cfg['FileName']+'_'+timeStamp+'.json', 'w') as outFile:
json.dump(self.jsonRaw,outFile)
outFile.close()
with open(self.pathOut+'FilteredTweets_'+self.cfg['FileName']+'_'+timeStamp+'.json', 'w') as outFile:
json.dump(meaningful,outFile)
outFile.close()
print 'Json text dump complete, buffering....'
time.sleep(1)
giSeeker.flushTweets(self)
else:
print "No tweets found for date"
print "Updating geoPickle"
updateGeoPickle(self.geoCache,self.cfg['Directory']+pickleName)
def saveTweets(self):
print "\nDumping tweets to file, contains %s tweets with %s accepted, %s rejected, %s partial matches, and %s irrelevant" % (self.cfg['StopCount'],
self.acceptedCount,
self.excludedCount,
self.partialCount,
self.irrelevantCount)
print '\tJson text dump complete....\n'
meaningful = self.jsonAccepted*self.cfg['KeepAccepted'] + self.jsonPartial*self.cfg['KeepPartial'] + self.jsonExcluded*self.cfg['KeepExcluded']
if self.cfg['TweetData'] != 'all':
meaningful = cleanJson(meaningful,self.cfg,self.tweetTypes)
timeStamp = self.startTime
if self.cfg['KeepRaw']:
with open(self.pathOut+'Raw_'+self.cfg['FileName']+'_'+timeStamp+'.json', 'w') as outFile:
json.dump(self.jsonRaw,outFile)
outFile.close()
with open(self.pathOut+'FilteredTweets_'+self.cfg['FileName']+'_'+timeStamp+'.json', 'w') as outFile:
json.dump(meaningful,outFile)
outFile.close()
giListener.flushTweets(self)
print "Updating geoPickle"
updateGeoPickle(self.geoCache,self.cfg['Directory']+pickleName)
def renewEventStagerStatus(self):
canFinish_file = os.path.join(self.__workDir, 'EventStagerStatusCan.json')
finished_file = os.path.join(self.__workDir, 'EventStagerStatus.json')
if self.__isDaemon:
if os.path.exists(canFinish_file):
#with open(canFinish_file) as inputFile:
# origin_status = json.load(inputFile)
# self.__canFinish = origin_status['canFinish']
self.__canFinish = True
if self.__status == "finished":
status = {'status': self.__status}
with open(finished_file, 'w') as outputFile:
json.dump(status, outputFile)
elif os.path.exists(finished_file):
os.remove(finished_file)
else:
if os.path.exists(finished_file):
#with open(finished_file) as inputFile:
# origin_status = json.load(inputFile)
# self.__status = origin_status['status']
self.__status = "finished"
if self.__canFinish:
status = {'canFinish': self.__canFinish}
with open(canFinish_file, 'w') as outputFile:
json.dump(status, outputFile)
elif os.path.exists(canFinish_file):
os.remove(canFinish_file)
def parse_duration_ffmpeg(media_url):
try:
with open(_MEDIA_FILE) as f:
_cache = json.load(f)
except IOError:
_cache = {}
except ValueError:
# the json file is corrupted
_cache = {}
if media_url not in _cache:
command = ['ffmpeg', '-i', media_url]
out, err = wrap_subprocess(command)
REGEX = re.compile('Duration: (\d+):(\d+):(\d+).(\d+)')
matches = REGEX.findall(err)
# if matches:
try:
found, = matches
except ValueError:
print err
return
hours = int(found[0])
minutes = int(found[1])
minutes += hours * 60
seconds = int(found[2])
seconds += minutes * 60
duration = seconds + minutes * 60 + hours * 60 * 60
_cache[media_url] = duration
with open(_MEDIA_FILE, 'w') as f:
json.dump(_cache, f, indent=4)
return _cache[media_url]
def main(argv):
global result, g_debug
parser = argparse.ArgumentParser(description="Convert LDIF file to json")
parser.add_argument("--input", help="input file", required=True)
parser.add_argument("--country_code", help="country code, e.g. +41", required=True)
parser.add_argument("--merge", help="file to merge with", default="out.json")
parser.add_argument('--debug', action='store_true')
args = parser.parse_args()
g_debug = args.debug
name = os.path.splitext(os.path.basename(args.merge))[0]
result = []
# merge
try:
data = open(args.merge, "r").read()
j = json.loads(data)
name = j["name"]
result = j["entries"]
#debug(result)
except IOError:
pass
# convert
parser = MyLDIF(open(args.input, "r"))
parser.parse()
result = cleanup_entries(result, args.country_code)
if len(result) != 0:
data = OrderedDict((
("name", name),
("entries", result)
))
with open(args.merge, 'w') as outfile:
json.dump(data, outfile, indent=2)
def _flatten_samples(samples, base_file):
"""Create a flattened JSON representation of data from the bcbio world map.
"""
out_file = "%s-samples.json" % utils.splitext_plus(base_file)[0]
flat_data = []
for data in samples:
cur_flat = {}
for key_path in [["analysis"], ["description"], ["rgnames"], ["config", "algorithm"],
["metadata"], ["genome_build"], ["resources"],
["files"], ["reference"], ["genome_resources"], ["vrn_file"]]:
cur_key = "__".join(key_path)
for flat_key, flat_val in _to_cwldata(cur_key, tz.get_in(key_path, data)):
cur_flat[flat_key] = flat_val
flat_data.append(cur_flat)
out = {}
for key in sorted(list(set(reduce(operator.add, [d.keys() for d in flat_data])))):
out[key] = []
for cur_flat in flat_data:
out[key].append(cur_flat.get(key))
# special case for back-compatibility with fasta specifications -- yuck
if "reference__fasta__base" not in out and "reference__fasta" in out:
out["reference__fasta__base"] = out["reference__fasta"]
del out["reference__fasta"]
with open(out_file, "w") as out_handle:
json.dump(out, out_handle, sort_keys=True, indent=4, separators=(',', ': '))
return out_file, _samplejson_to_inputs(out), out
def run(self):
self.correct_octave_errors_by_chunks()
self.remove_extreme_values()
self.correct_jumps()
self.pitch = list(reversed(self.pitch))
self.correct_jumps()
self.pitch = list(reversed(self.pitch))
self.filter_noise_region()
self.correct_oct_error()
self.pitch = list(reversed(self.pitch))
self.correct_oct_error()
self.pitch = list(reversed(self.pitch))
self.correct_octave_errors_by_chunks()
self.filter_chunks_by_energy(chunk_limit=60)
if self.out:
self.data['pitch'] = self.pitch
with open(self.fname[:-5] + "_filtered.json", 'w') as f: json.dump(self.data, f)
return self.pitch
def SaveScene(file, quad_tree, extra_data=None):
export_data = []
if extra_data:
for item in extra_data:
export_data.append(item)
for node in quad_tree:
for child in node.getChildren():
temp = {}
if child.hasPythonTag("actor_files"):
temp["actor"] = unicode(child.getPythonTag("actor_files")[0])
temp["actor_anims"] = child.getPythonTag("actor_files")[1]
temp["actor_collision"] = child.getPythonTag("actor_files")[2]
elif child.hasPythonTag("model_file"):
temp["model"] = unicode(child.getPythonTag("model_file"))
if child.hasPythonTag("light_color"):
c = child.getPythonTag("light_color")
temp["color_r"] = c[0]
temp["color_g"] = c[1]
temp["color_b"] = c[2]
temp["rotation_h"] = child.getH(render)
temp["rotation_p"] = child.getP(render)
temp["rotation_r"] = child.getR(render)
temp["position_x"] = child.getX(render)
temp["position_y"] = child.getY(render)
temp["position_z"] = child.getZ(render)
temp["scale"] = child.getScale()[0]
temp["parent_name"] = node.getName()
temp["parent_index"] = quad_tree.index(node)
temp["props"] = unicode(child.getPythonTag("props"))
export_data.append(temp)
with open(file, "w") as outfile:
json.dump(export_data, outfile, indent=4, separators=(",", ": "), sort_keys=True)
def dump_results(specs, output_path):
with open(output_path, 'w') as output_file:
results = OrderedDict([
("env", ENV),
("specs", specs)
])
json.dump(results, output_file, indent=4)
def main():
my_config = get_config()
new_synapse_config = generate_configuration(
my_config, get_zookeeper_topology(), get_all_namespaces()
)
with tempfile.NamedTemporaryFile() as tmp_file:
new_synapse_config_path = tmp_file.name
with open(new_synapse_config_path, 'w') as fp:
json.dump(new_synapse_config, fp, sort_keys=True, indent=4, separators=(',', ': '))
# Match permissions that puppet expects
os.chmod(new_synapse_config_path, 0644)
# Restart synapse if the config files differ
should_restart = not filecmp.cmp(new_synapse_config_path, my_config['config_file'])
# Always swap new config file into place. Our monitoring system
# checks the config['config_file'] file age to ensure that it is
# continually being updated.
shutil.copy(new_synapse_config_path, my_config['config_file'])
if should_restart:
subprocess.check_call(SYNAPSE_RESTART_COMMAND)
def get_filenames_with_industry(link):
listed_company={}
html=requests.get(link)
soup1=BeautifulSoup(html.content)
try:
with open("url_industry_list.json","r") as outfile:
listed_company=json.load(outfile)
outfile.close()
except:
print "file : url_industry_list does not exist"
with open("url_industry_list.json","w") as outfile:
for a in soup1.findAll('td'):
if a.findChild():
l=str(a.findChild().get('href'))
if not nasdaq.match(l) and linkPattern.match(l):
url=str(a.findChild().get('href'))
filname=url.replace('http://','')
filname=filname.replace( 'https://','')
filname=filname.replace('www.','')
# filname=filname.replace('.com','')
# filname=filname.replace( '.org','')
# filname=filname.replace( '.gov','')
filname=re.sub('\..*|\/.*','',filname)
ind=str(a.findNextSiblings(style='width:105px')[0].get_text())
#outfile.write(l+'\n')
if filname not in listed_company:
listed_company[filname]=ind
json.dump(listed_company , outfile)
#return listed_company
print listed_company
def flush(self):
file = open(self.path, "wb")
json.dump(self.pagesData, file)
file.close()
for page in self.pages:
self.pages[page].flush()
def saveAnnotation(self):
f = codecs.open((self.filename + '.info'), 'w', 'utf8')
json.dump(self.annotation,
f,
ensure_ascii=False,
indent=4)
f.close()
def log_metric(self, name, value, unit=None, global_step=None, extras=None):
"""Log the benchmark metric information to local file.
Currently the logging is done in a synchronized way. This should be updated
to log asynchronously.
Args:
name: string, the name of the metric to log.
value: number, the value of the metric. The value will not be logged if it
is not a number type.
unit: string, the unit of the metric, E.g "image per second".
global_step: int, the global_step when the metric is logged.
extras: map of string:string, the extra information about the metric.
"""
if not isinstance(value, numbers.Number):
tf.logging.warning(
"Metric value to log should be a number. Got %s", type(value))
return
with tf.gfile.GFile(
os.path.join(self._logging_dir, _METRIC_LOG_FILE_NAME), "a") as f:
metric = {
"name": name,
"value": float(value),
"unit": unit,
"global_step": global_step,
"timestamp": datetime.datetime.now().strftime(
_DATE_TIME_FORMAT_PATTERN),
"extras": extras}
try:
json.dump(metric, f)
f.write("\n")
except (TypeError, ValueError) as e:
tf.logging.warning("Failed to dump metric to log file: "
"name %s, value %s, error %s", name, value, e)
def json_dump(self, filename="", indent=4, sort_keys=True):
if filename == "":
filename = "%s.json" % self.instance_name
with open(filename, "w") as f:
json.dump(self.to_dict(), f, indent=indent, sort_keys=sort_keys)
def cache(running_jobs):
# dump to "cache"
with open(NOTEBOOK_CACHE, 'w') as fp:
json.dump(running_jobs, fp)
def build_new_cnn_relation(train_file, user_feature_file, user_follow_file, out_dir):
user_url_dict = {}
url_user_dict = {}
with open(train_file) as f:
lines = f.readlines()
for line in lines:
uid_no, url_no, freq, ts = line.strip().split()
if int(uid_no) not in user_url_dict:
user_url_dict[int(uid_no)] = []
user_url_dict[int(uid_no)].append(int(url_no))
if int(url_no) not in url_user_dict:
url_user_dict[int(url_no)] = []
url_user_dict[int(url_no)].append(int(uid_no))
print('user url dict: %d, url user dict: %d' % (len(user_url_dict), len(url_user_dict)))
user2idx_dict = {}
with open(user_feature_file) as f:
user_feature_list = json.loads(f.read())
assert len(user_feature_list) == 11576
##User level: ID, # of followers, # of following, posts urls, post cnt, favorite category, favorite website, uid, name
for item in user_feature_list:
# item[7] is string
user2idx_dict[item[7]] = int(item[0])
uid_mapped_file = '/home/dyou/url_recsys/data_gen/data_181110/uid_mapped.txt'
with open(uid_mapped_file) as f:
lines = f.readlines()
for line in lines:
#0 767036004 upayr
info = [item.strip() for item in line.strip().split()]
assert len(info) == 3
uid_no, uid = info[0], info[1]
assert int(uid_no) == user2idx_dict[uid]
print('check uid mapped success')
with open(user_follow_file) as f:
lines = f.readlines()
ori_follower_dict = {}
ori_follow_in_dict = {}
for line in lines[1:]:
#follower,followee followee = follow in
#757778071734804480,815708250217840640
follower, follow_in = line.strip().split(',')
if follower not in ori_follower_dict:
ori_follower_dict[follower] = []
if follow_in not in ori_follower_dict[follower]:
ori_follower_dict[follower].append(follow_in)
if follow_in not in ori_follow_in_dict:
ori_follow_in_dict[follow_in] = []
if follower not in ori_follow_in_dict[follow_in]:
ori_follow_in_dict[follow_in].append(follower)
miss_follower, miss_follow_in = 0, 0
uid_no_follower_dict, uid_no_follow_in_dict = {}, {}
for uid in user2idx_dict:
if uid not in ori_follower_dict:
miss_follower += 1
uid_no_follower_dict[int(user2idx_dict[uid])] = []
else:
res_list = []
for uid2 in ori_follower_dict[uid]:
if uid2 in user2idx_dict:
res_list.append(int(user2idx_dict[uid2]))
if len(res_list) == 0:
miss_follower += 1
uid_no_follower_dict[int(user2idx_dict[uid])] = []
else:
uid_no_follower_dict[int(user2idx_dict[uid])] = res_list
if uid not in ori_follow_in_dict:
miss_follow_in += 1
uid_no_follow_in_dict[int(user2idx_dict[uid])] = []
else:
res_list = []
for uid2 in ori_follow_in_dict[uid]:
if uid2 in user2idx_dict:
res_list.append(int(user2idx_dict[uid2]))
if len(res_list) == 0:
miss_follow_in += 1
uid_no_follow_in_dict[int(user2idx_dict[uid])] = []
else:
uid_no_follow_in_dict[int(user2idx_dict[uid])] = res_list
print('follow users: %d, miss follower: %d, follow in users: %d, miss follow in: %d'
% (len(uid_no_follower_dict), miss_follower, len(uid_no_follow_in_dict), miss_follow_in))
url_neighb_urls = {}
for url_no in url_user_dict:
direct_user_list = url_user_dict[url_no]
two_step_url_list = []
for user_no in direct_user_list:
two_step_url_list.extend(user_url_dict[user_no])
two_step_url_set = set(two_step_url_list)
two_step_url_set.remove(url_no)
url_neighb_urls[url_no] = list(two_step_url_set)
print('url_neighb_urls:', len(url_neighb_urls))
user_neighb_users = {}
for uid_no in user_url_dict:
direct_url_list = user_url_dict[uid_no]
two_step_user_list = []
for url_no in direct_url_list:
two_step_user_list.extend(url_user_dict[url_no])
two_step_user_set = set(two_step_user_list)
two_step_user_set.remove(uid_no)
user_neighb_users[uid_no] = list(two_step_user_set)
print('user_neighb_users:', len(user_neighb_users))
json.dump(user2idx_dict, open(out_dir + os.sep + 'user2idx.json', 'w'))
json.dump(ori_follower_dict, open(out_dir + os.sep + 'ori_follower_dict.json', 'w'))
json.dump(ori_follow_in_dict, open(out_dir + os.sep + 'ori_follow_in_dict.json', 'w'))
json.dump(user_url_dict, open(out_dir + os.sep + 'user_url_dict.json', 'w'))
json.dump(url_user_dict, open(out_dir + os.sep + 'url_user_dict.json', 'w'))
json.dump(url_neighb_urls, open(out_dir + os.sep + 'url_neighb_urls.json', 'w'))
json.dump(user_neighb_users, open(out_dir + os.sep + 'user_neighb_users.json', 'w'))
json.dump(uid_no_follower_dict, open(out_dir + os.sep + 'uid_no_follower_dict.json', 'w'))
json.dump(uid_no_follow_in_dict, open(out_dir + os.sep + 'uid_no_follow_in_dict.json', 'w'))
print('new cnn relation done')
def main():
with open('step1/results1/tagging_1.json') as f:
part1=json.load(f)
with open('step1/results2/tagging_2.json') as f:
part2=json.load(f)
with open('step1/results3/tagging_3.json') as f:
part3=json.load(f)
with open('step1/results4/tagging_4.json') as f:
part4=json.load(f)
with open('step1/results5/tagging_5.json') as f:
part5=json.load(f)
all_label=[]
with open('../dataset/label_id.txt','r') as f:
for line in f.readlines():
label,_ = line.split('\t')
all_label.append(label)
all_label_set = set(all_label)
for file in part1.keys():
label_set = set(part1[file]['result'][0]['labels'])
res_label = list(all_label_set - label_set)
part1[file]['result'][0]['labels'] += res_label
part1[file]['result'][0]['scores'] += ['0']*32
assert len(res_label)==32
assert len(part1[file]['result'][0]['labels'])==82
for file in part2.keys():
label_set = set(part2[file]['result'][0]['labels'])
res_label = list(all_label_set - label_set)
part2[file]['result'][0]['labels'] += res_label
part2[file]['result'][0]['scores'] += ['0']*32
assert len(res_label)==32
assert len(part2[file]['result'][0]['labels'])==82
for file in part3.keys():
label_set = set(part3[file]['result'][0]['labels'])
res_label = list(all_label_set - label_set)
part3[file]['result'][0]['labels'] += res_label
part3[file]['result'][0]['scores'] += ['0']*32
assert len(res_label)==32
assert len(part3[file]['result'][0]['labels'])==82
for file in part4.keys():
label_set = set(part4[file]['result'][0]['labels'])
res_label = list(all_label_set - label_set)
part4[file]['result'][0]['labels'] += res_label
part4[file]['result'][0]['scores'] += ['0']*32
assert len(res_label)==32
assert len(part4[file]['result'][0]['labels'])==82
for file in part5.keys():
label_set = set(part5[file]['result'][0]['labels'])
res_label = list(all_label_set - label_set)
part5[file]['result'][0]['labels'] += res_label
part5[file]['result'][0]['scores'] += ['0']*32
assert len(res_label)==32
assert len(part5[file]['result'][0]['labels'])==82
w1 = 0.2
w2 = 0.2
w3 = 0.2
w4 = 0.2
w5 = 0.2
av_result = {}
for file in part1.keys():
av_result[file] = {}
av_result[file]['result'] = []
av_result[file]['result'].append({})
av_result[file]['result'][0]['labels'] = []
av_result[file]['result'][0]['scores'] = []
for index, label in enumerate(part1[file]['result'][0]['labels']):
av_result[file]['result'][0]['labels'].append(label)
index1 = part1[file]['result'][0]['labels'].index(label)
index2 = part2[file]['result'][0]['labels'].index(label)
index3 = part3[file]['result'][0]['labels'].index(label)
index4 = part4[file]['result'][0]['labels'].index(label)
index5 = part5[file]['result'][0]['labels'].index(label)
score = (w1*eval(part1[file]['result'][0]['scores'][index1]) +
w2*eval(part2[file]['result'][0]['scores'][index2]) +
w3*eval(part3[file]['result'][0]['scores'][index3]) +
w4*eval(part4[file]['result'][0]['scores'][index4]) +
w5*eval(part5[file]['result'][0]['scores'][index5]) )
av_result[file]['result'][0]['scores'].append("%.4f" % score)
for file in av_result.keys():
score = av_result[file]['result'][0]['scores']
label = av_result[file]['result'][0]['labels']
sorted_score = sorted(enumerate(score), key=lambda x: x[1],reverse=True)
idx = [i[0] for i in sorted_score]
top20_index=idx[:20]
av_result[file]['result'][0]['scores'] =[score[i] for i in top20_index]
av_result[file]['result'][0]['labels'] =[label[i] for i in top20_index]
with open('step1/results1/train_av_result.json', 'w', encoding="utf-8") as f:
json.dump(av_result, f, ensure_ascii=False, indent=4)
train_label=pd.read_csv('../dataset/tagging/GroundTruth/tagging_info.txt',header=None,names=['video','label'],sep='\t')
predict_df=pd.read_json('step1/results1/train_av_result.json', orient='index')
predict_df['label']=predict_df.apply(lambda row:row['result'][0]['labels'],axis=1)
predict_df=predict_df.reset_index()
predict_df.rename(columns={'index':'video'},inplace=True)
predict_df['label_score']=predict_df.apply(lambda row:row['result'][0]['scores'],axis=1)
predict_df.drop('result',axis=1,inplace=True)
predict_df['soft_label'] = predict_df.apply(lambda row:[0.]*82,axis=1)
class Preprocess_label_sparse_to_dense:
def __init__(self,
index_dict,
sep_token=',',
is_training=False):
self.index_to_tag,self.tag_to_index = extract_dict(index_dict)
self.sep_token = sep_token
self.is_training = is_training
self.max_index = 0
for index in self.index_to_tag:
self.max_index = max(index, self.max_index)
self.seq_size = self.max_index + 1
self.label_num = self.seq_size
def __call__(self, index_str):
dense_array = np.zeros(self.seq_size)
label_lst = index_str.split(self.sep_token)
for label in label_lst:
if label in self.tag_to_index:
index = self.tag_to_index[label]
dense_array[index] = 1.0
return dense_array.astype('float32')
def extract_dict(dict_file):
index_to_tag = {}
tag_to_index = {}
for i, line in enumerate(codecs.open(dict_file, 'r', encoding='utf-8')):
line = line.strip()
if '\t' in line:
index, tag = line.split('\t')[:2]
elif ' ' in line:
index, tag = i, line.rsplit(' ', 1)[0]
else:
index, tag = i, line
try:
index = int(index)
except:
index, tag = int(tag), index
index_to_tag[index] = tag
tag_to_index[tag] = index
return index_to_tag, tag_to_index
preprocess_instance=Preprocess_label_sparse_to_dense('../dataset/label_id.txt')
tag_to_index = preprocess_instance.tag_to_index
predict_df['label_index']=predict_df.apply(lambda row:[tag_to_index[tag] for tag in row['label']],axis=1)
for i in range(predict_df.shape[0]):
temp_list = np.array(predict_df.iloc[i]['soft_label'],dtype=np.float)
temp_list[predict_df.iloc[i]['label_index']] = predict_df.iloc[i]['label_score']
update_row=dict(predict_df.iloc[i])
update_row['soft_label']= temp_list.tolist()
predict_df.iloc[i] = pd.Series(update_row)
predict_dfV1 = predict_df[['video','soft_label']]
predict_dfV1.to_csv('./train_soft_label.csv',index=None)
retriever = DensePassageRetriever(document_store=document_store,
query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
use_gpu=True,
batch_size=256,
embed_title=True)
# apparently this is time consuming
print("Updating embeddings...'")
document_store.update_embeddings(retriever)
document_store.save("wiki_dump_embeddings.faiss")
# example:
# retrieved_doc = retriever.retrieve(query="Why did the revenue increase?")
# tune these parameters too
print("Running QA Reader")
reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2",
use_gpu=True, no_ans_boost=-10, context_window_size=500,
top_k_per_candidate=3, top_k_per_sample=1,
num_processes=1, max_seq_len=256, doc_stride=128)
# example:
# reader.predict(question="Who is the father of Arya Starck?", documents=retrieved_docs, top_k=3)
print("Started pipeline")
p = Pipeline()
p.add_node(component=retriever, name="ESRetriever1", inputs=["Query"])
p.add_node(component=reader, name="QAReader", inputs=["ESRetriever1"])
res = p.run(query="What did Einstein work on?", params={"retriever": {"top_k": 1}})
json.dump(res, open('answer.json', 'w'), default=dumper)
exit()
for n in range(min_n,min(max_n + 1, n_original_tokens + 1)):
for i in range(n_original_tokens - n + 1):
tokens.append(" ".join(original_tokens[i: i + n]))
return tokens
finalcloud=[]
for i in range(len(wlist)):
preob = {}
preob["daysfromRelease"]=wlist[i].get("daySince")
print(wlist[i].get("daySince"))
prep = wlist[i].get("cloud")
# print(prep)
cut = jieba.cut(str(prep))
listcut = list(cut)
# print(listcut)
n_gramWords = _word_ngrams(tokens = listcut,ngram_range=(1,1))
preob["finalcloud"] = n_gramWords
finalcloud.append(preob)
# print(finalcloud)
print('saving as json')
with open('dawn.json', 'w', encoding="utf-8") as f:
json.dump(finalcloud, f, indent=4, ensure_ascii=False)
# for n_gramWord in n_gramWords:
# print(n_gramWord)
# 版权声明：本文为CSDN博主「姚贤贤」的原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接及本声明。
# 原文链接：https://blog.csdn.net/u011311291/article/details/79269931
def write_data_export_zip(self, request, response):
with zipfile.ZipFile(response, mode="w") as zip_file:
with zip_file.open(
f"{self.slug}.json", mode="w"
) as json_binary_file, io.TextIOWrapper(json_binary_file) as json_file:
json.dump(
{
"email": self.user.email,
"date_joined": self.user.date_joined.isoformat(),
"last_login": self.user.last_login.isoformat(),
"url": request.build_absolute_uri(self.get_absolute_url()),
"is_public": self.is_public,
"name": self.name,
"city_or_town": self.city_or_town,
"country": self.country,
"cause_areas": list(map(CauseArea.label, self.cause_areas)),
"cause_areas_other": self.cause_areas_other,
"available_to_volunteer": self.available_to_volunteer,
"open_to_job_offers": self.open_to_job_offers,
"expertise_areas": list(
map(ExpertiseArea.label, self.expertise_areas)
),
"expertise_areas_other": self.expertise_areas_other,
"career_interest_areas": list(
map(ExpertiseArea.label, self.career_interest_areas)
),
"available_as_speaker": self.available_as_speaker,
"topics_i_speak_about": self.topics_i_speak_about,
"organisational_affiliations": list(
map(
OrganisationalAffiliation.label,
self.organisational_affiliations,
)
),
"summary": self.summary,
"giving_pledges": list(
map(GivingPledge.label, self.giving_pledges)
),
"member_of_local_groups": [
request.build_absolute_uri(local_group.get_absolute_uri())
for local_group in self.local_groups.all()
],
"organiser_of_local_groups": [
request.build_absolute_uri(local_group.get_absolute_uri())
for local_group in self.user.localgroup_set.all()
],
"aliases": [
request.build_absolute_uri(
urls.reverse("profile", kwargs={"slug": slug.slug})
)
for slug in self.slugs.filter(redirect=True)
],
"legacy_hub_url": (
self.legacy_record
and request.build_absolute_uri(
urls.reverse(
"profile_legacy",
kwargs={"legacy_record": self.legacy_record},
)
)
),
},
json_file,
indent=2,
)
if self.image:
with self.image.open() as image_src_file, zip_file.open(
self.slug + pathlib.PurePath(self.image.name).suffix, mode="w"
) as image_dst_file:
shutil.copyfileobj(image_src_file, image_dst_file)
cv["cv%02d/all" % (cv_outer_i)] = [tr_val, te]
cv_inner = StratifiedKFold(y[tr_val].ravel(),
n_folds=NFOLDS_INNER,
random_state=42)
for cv_inner_i, (tr, val) in enumerate(cv_inner):
cv["cv%02d/cvnested%02d" %
((cv_outer_i), cv_inner_i)] = [tr_val[tr], tr_val[val]]
for k in cv:
cv[k] = [cv[k][0].tolist(), cv[k][1].tolist()]
C_range = [[100], [10], [1], [1e-1], [1e-2], [1e-3], [1e-4], [1e-5],
[1e-6], [1e-7], [1e-8], [1e-9]]
user_func_filename = "/home/ad247405/git/scripts/2016_schizConnect/supervised_analysis/all_studies+VIP/all_subjects/Freesurfer/ROIs/learning_curve_ratios/intra_site/svm_ratio_0_2.py"
config = dict(data=dict(X="Xrois_vol+cov.npy", y="y.npy"),
params=C_range,
resample=cv,
map_output="model_selectionCV",
user_func=user_func_filename,
reduce_input="results/*/*",
reduce_group_by="params",
reduce_output="model_selectionCV.csv")
json.dump(config, open(os.path.join(WD, "config_dCV.json"), "w"))
# Build utils files: sync (push/pull) and PBS
import brainomics.cluster_gabriel as clust_utils
sync_push_filename, sync_pull_filename, WD_CLUSTER = \
clust_utils.gabriel_make_sync_data_files(WD)
cmd = "mapreduce.py --map %s/config_dCV.json" % WD_CLUSTER
clust_utils.gabriel_make_qsub_job_files(WD, cmd)
def write_json(data: dict, dirname: str, subname: str):
filepath = prepare_filepath(dirname, subname, JSON_EXT)
LOG.debug(f"Writing JSON file: {filepath}")
with open(filepath, "w") as fp:
json.dump(data, fp, indent=2, sort_keys=True)
dbroot = 'jsondb/'
for arg in sys.argv:
if (argi > 0):
print "// Debug: parsing " + arg + "\n"
tree = ElementTree.parse(arg)
root = tree.getroot()
if root.tag != 'application':
print "ERROR: XML root element is <%s> but expected <application>" % root.tag
sys.exit(2)
for objType in root.findall('objType'):
typeName = objType.get('name')
objects = []
fp = open(dbroot + typeName + '.json', 'w')
if fp:
print "Generating initial database for %s" % typeName
id = 1
for obj in root.findall('object'):
if (obj.get('type') == typeName):
properties = dict(obj.attrib)
properties['id'] = id
for child in obj.findall('*'):
properties[child.tag] = child.text
objects.append(properties)
id += 1
if len(objects) != 0:
json.dump(objects, fp)
fp.close()
argi += 1
def write_min_json(data: dict, dirname: str, subname: str):
filepath = prepare_filepath(dirname, subname, MIN_JSON_EXT)
LOG.debug(f"Writing minified JSON file: {filepath}")
with open(filepath, "w") as fp:
json.dump(data, fp, separators=(",", ":"), sort_keys=True)
entry['cases'] += 1
entry_county = entry['counties'][p_county]
entry_county['cases'] += 1
entry_county['age'][person[0]] += 1
else:
skipped += 1
if unknown_in_fille > 0:
print '# of unknown people:', unknown_in_fille
t1 = time.clock()
print '\t',n,'records in',(t1-t0),' secs'
if len(unknown) > 0:
print 'unknown: ',len(unknown)
with open('unknown.txt', 'wb') as f:
for item in unknown:
print>>f, item
print 'save json'
t0 = time.clock()
with open(os.path.join(dir_name, 'cases_per_day.json'), 'wb') as f:
json.dump(cases_per_day, f, sort_keys=True) #indent=2,
t1 = time.clock()
print'\t', (t1-t0), ' secs'
print 'done'
# if skipped > 0:
# print '\nskipped:', skipped
def save_json(fname, data):
with open(fname, "w") as json_file:
json.dump(data, json_file, indent=4, sort_keys=True)
def test( total_k_step, is_mc ):
with tf.Graph().as_default():
loader = data_loader.data_loader(
batch_size=BATCH_SIZE,
dataset_path=PREPRO_DATASET_JSON,
qa_data_path=PREPRO_QA_H5,
img_tfrecord_size=IMG_TFRECORD_SIZE,
max_words_q=MAX_WORDS_Q,
dim_img=DIM_IMG,
is_training=False )
num_test = loader.get_test_length()
vocab_size = loader.get_vocabulary_size()
answer_vocab_size = loader.get_answer_vocab_size()
fname = 'eval_result/MultipleChoice_mscoco_val2014_dan_results.json' if is_mc else 'eval_result/OpenEnded_mscoco_val2014_dan_results.json'
print 'total questions: ' + str(num_test)
print 'vocabulary_size : ' + str(vocab_size)
print 'file to write: ', fname
with tf.Session() as sess:
print 'constructing model...'
model = DAN.Dual_Attention_Network(
scope='dan',
sess=sess,
batch_size=BATCH_SIZE,
k_step=total_k_step,
dim_img=DIM_IMG,
max_words_q=MAX_WORDS_Q,
vocab_size=vocab_size,
vocab_embed_size=VOCAB_EMBED_SIZE,
rnn_size=RNN_SIZE,
hidden_size=HIDDEN_SIZE,
ans_num=answer_vocab_size,
dan_npy_path=DAN_VAR_PATH)
print 'model built'
print 'initialize global variables..'
sess.run(tf.global_variables_initializer())
tf.train.start_queue_runners(sess=sess)
qa_dict = {}
result = []
tStart_total = time.time()
for itr in range(0, num_test, BATCH_SIZE):
idx_batch, ques_id_batch, question_batch, ques_len_batch, img_idx_batch, img_feat_batch, MC_ans_batch = loader.get_next_test_batch(sess)
predicted = model.predict_answers( img_feat_batch, question_batch, ques_len_batch, 1.0)
loader.print_test_example( img_idx_batch[0], question_batch[0], predicted[0] )
avn1 = model.get_avn(1, img_feat_batch, question_batch, ques_len_batch, 1.0)
avn2 = model.get_avn(2, img_feat_batch, question_batch, ques_len_batch, 1.0)
avn_list = [avn1[0], avn2[0]]
aut1 = model.get_aut(1, img_feat_batch, question_batch, ques_len_batch, 1.0)
aut2 = model.get_aut(2, img_feat_batch, question_batch, ques_len_batch, 1.0)
aut_list = [aut1[0], aut2[0]]
loader.save_test_example( ques_id_batch[0], img_idx_batch[0], avn_list, question_batch[0], aut_list, predicted[0] )
if is_mc is False:
# for open-ended
top_ans = np.argmax(predicted, axis=1)
else:
# for multiple-choice
for b in range(BATCH_SIZE):
predicted[b, MC_ans_batch[b]] += 1
top_ans = np.argmax(predicted, axis=1)
for i in range(0,BATCH_SIZE):
ans = loader.get_answer( top_ans[i] )
if int(ques_id_batch[i]) not in qa_dict:
qa_dict[int(ques_id_batch[i])] = ans
if itr % 300*BATCH_SIZE ==0:
print "Testing batch: ", itr
print "Testing done."
tStop_total = time.time()
print "Total Time Cost:", round(tStop_total - tStart_total,2), "s"
print 'Saving result...'
for key in qa_dict.keys():
result.append({ u'question_id': key, u'answer': qa_dict.pop(key) })
print "total qa pairs: ", len(result)
fname = 'eval_result/MultipleChoice_mscoco_val2014_dan_results.json' if is_mc else 'eval_result/OpenEnded_mscoco_val2014_dan_results.json'
print 'wirte %s'% fname
json.dump(result,open(fname,'w'))
ch += 4 * ' ' + '<title lang="ar">' + tit.replace(
''', "'").replace('"', '"').replace(
'&', 'و'.decode('utf-8')).replace(
'<div class="schedule__row__nowshowing">',
'') + '</title>\n'
ch += 4 * ' ' + '<desc lang="ar">' + de.replace(
''', "'").replace('"', '"').replace(
'&',
'و'.decode('utf-8')).strip() + '</desc>\n </programme>\r'
with io.open("/etc/epgimport/aljazeera.xml", "a",
encoding="utf-8") as f:
f.write(ch)
print 'aljazeera epg download finished'
else:
'No data found for aljazeera'
with io.open("/etc/epgimport/aljazeera.xml", "a", encoding="utf-8") as f:
f.write(('</tv>').decode('utf-8'))
import json
with open('/usr/lib/enigma2/python/Plugins/Extensions/Epg_Plugin/times.json',
'r') as f:
data = json.load(f)
for channel in data['bouquets']:
if channel["bouquet"] == "aljazeera":
channel['date'] = datetime.today().strftime('%A %d %B %Y at %I:%M %p')
with open('/usr/lib/enigma2/python/Plugins/Extensions/Epg_Plugin/times.json',
'w') as f:
json.dump(data, f)
def save_json(data, filename):
import json
with open(filename, 'w') as f:
json.dump(data, f, indent=4)
def main(args):
cfg = setup(args)
logger = setup_logger(output=cfg.OUTPUT_DIR, distributed_rank=comm.get_rank(), name="imbalance detection")
detector_ours = build_detector(cfg)
gambler_ours = build_gambler(cfg)
DetectionCheckpointer(detector_ours, save_dir=cfg.OUTPUT_DIR).resume_or_load(
os.path.join(args.dir_ours, "model_0099999.pth"), resume=args.resume
)
# DetectionCheckpointer(gambler_ours, save_dir=cfg.OUTPUT_DIR).resume_or_load(
# os.path.join(args.dir_ours, "gambler_models/model_final.pth"), resume=args.resume
# )
detector_baseline = build_detector(cfg)
gambler_baseline = build_gambler(cfg)
DetectionCheckpointer(detector_baseline, save_dir=cfg.OUTPUT_DIR).resume_or_load(
os.path.join(args.dir_baseline, "model_final.pth"), resume=args.resume
)
# DetectionCheckpointer(gambler_baseline, save_dir=cfg.OUTPUT_DIR).resume_or_load(
# cfg.MODEL.GAMBLER_HEAD.WEIGHTS, resume=args.resume
# )
dicts = list(DatasetCatalog.get(args.dataset))
metadata = MetadataCatalog.get(args.dataset)
for idx, dataset_name in enumerate(cfg.DATASETS.TEST):
test_data_loader = GANTrainer.build_test_loader(cfg, dataset_name)
train_data_loader = GANTrainer.build_train_loader(cfg)
evaluator_ours = GANTrainer.build_evaluator(cfg, dataset_name, os.path.join(args.dir_ours, "inference"))
evaluator_baseline = GANTrainer.build_evaluator(cfg, dataset_name, os.path.join(args.dir_baseline, "inference"))
analyzer = Analyzer(args.output)
# visualize_inference(detector, gambler, train_data_loader, args.source)
if args.per_image_ap:
# import json
# file_path = os.path.join("./output/gambler/2_full_coco/smoothing/fromscratch/focal_noimage/inference",
# "coco_instances_results_mine.json")
# with PathManager.open(file_path, "r") as f:
# imgid_to_pred_ours = json.load(f)
#
# from collections import defaultdict
# _imgid_to_pred = defaultdict(list)
# for prediction in imgid_to_pred_ours:
# _imgid_to_pred[prediction["image_id"]].append(prediction)
# imgid_to_pred_ours = _imgid_to_pred
#
# file_path = os.path.join(
# "./output/retinanet_baselines/simple_retinanet/simple_retina_fullcoco/all_layers_3scales/inference",
# "coco_instances_results.json")
# with PathManager.open(file_path, "r") as f:
# imgid_to_pred_base = json.load(f)
#
# from collections import defaultdict
# _imgid_to_pred_base = defaultdict(list)
# for prediction in imgid_to_pred_base:
# _imgid_to_pred_base[prediction["image_id"]].append(prediction)
# imgid_to_pred_base = _imgid_to_pred_base
#
# # imgid_to_topk_APs={k: v for k, v in sorted(imgid_to_topk_APs.items(), key=lambda item: item[1])}
# args.conf_threshold = 0.2
# a = dict(islice(imgid_to_topk_APs.items(), 200))
#
# for dic in tqdm.tqdm(dicts):
# if dic["image_id"] in a.keys():
# img = cv2.imread(dic["file_name"], cv2.IMREAD_COLOR)[:, :, ::-1]
# basename = os.path.basename(dic["file_name"])
# predictions_ours = create_instances(imgid_to_pred_ours[dic["image_id"]], img.shape[:2])
# # predictions_ours.remove("pred_masks")
# vis = Visualizer(img, metadata)
# vis_pred = vis.draw_instance_predictions(predictions_ours).get_image()
#
# vis = Visualizer(img, metadata)
# vis_gt = vis.draw_dataset_dict(dic).get_image()
#
# predictions_base = create_instances(imgid_to_pred_base[dic["image_id"]], img.shape[:2])
# # predictions_base.remove("pred_masks")
# vis = Visualizer(img, metadata)
# vis_pred_baseline = vis.draw_instance_predictions(predictions_base).get_image()
#
# concat = np.concatenate((vis_pred, vis_pred_baseline, vis_gt), axis=1)
# cv2.imwrite(os.path.join(args.output, basename), concat[:, :, ::-1])
with EventStorage(0) as storage:
analyzer.find_ap_per_img(detector_ours, test_data_loader, evaluator_ours)
imgid_to_ap_ours, imgid_to_pred_ours = analyzer.imgid_to_AP, analyzer.imgid_to_pred
analyzer.save()
analyzer.find_ap_per_img(detector_baseline, test_data_loader, evaluator_baseline)
imgid_to_ap_base, imgid_to_pred_base = analyzer.imgid_to_AP, analyzer.imgid_to_pred
analyzer.save()
imgid_to_topk_APs = get_topk_different_imgs(imgid_to_ap_ours, imgid_to_ap_base, k=args.k, order=args.sort_order)
if args.sort_order == "desc":
logger.info(f"The topk most different APs where our model outperformed")
elif args.sort_order == "asc":
logger.info(f"The topk most different APs where baseline outperformed")
for dic in tqdm.tqdm(dicts):
if dic["image_id"] in imgid_to_topk_APs.keys():
img = cv2.imread(dic["file_name"], cv2.IMREAD_COLOR)[:, :, ::-1]
basename = os.path.basename(dic["file_name"])
predictions_ours = create_instances(imgid_to_pred_ours[dic["image_id"]], img.shape[:2])
predictions_ours.remove("pred_masks")
vis = Visualizer(img, metadata)
vis_pred = vis.draw_instance_predictions(predictions_ours).get_image()
vis = Visualizer(img, metadata)
vis_gt = vis.draw_dataset_dict(dic).get_image()
predictions_base = create_instances(imgid_to_pred_base[dic["image_id"]], img.shape[:2])
predictions_base.remove("pred_masks")
vis = Visualizer(img, metadata)
vis_pred_baseline = vis.draw_instance_predictions(predictions_base).get_image()
concat = np.concatenate((vis_pred, vis_pred_baseline, vis_gt), axis=1)
cv2.imwrite(os.path.join(args.output, basename), concat[:, :, ::-1])
if args.compare_aps:
with EventStorage(0) as storage:
if os.path.isfile(os.path.join(args.dir_baseline, "results.json")):
with PathManager.open(os.path.join(args.dir_baseline, "results.json"), "r") as f:
results_baseline = json.load(f)
else:
results_baseline = load_old_inference_results(test_data_loader, evaluator_baseline)
with PathManager.open(os.path.join(args.dir_baseline, "results.json"), "w") as f:
json.dump(results_baseline, f, indent=2)
if os.path.isfile(os.path.join(args.dir_ours, "results.json")):
with PathManager.open(os.path.join(args.dir_ours, "results.json"), "r") as f:
results_ours = json.load(f)
else:
results_ours = load_old_inference_results(test_data_loader, evaluator_ours)
with PathManager.open(os.path.join(args.dir_ours, "results.json"), "w") as f:
json.dump(results_ours, f, indent=2)
plot_aps(cfg, args, results_ours, results_baseline, sort=args.sort)
# results_ours_dict = OrderedDict()
# results_baseline_dict = OrderedDict()
#
# results_ours_dict[dataset_name] = results_ours
# results_baseline_dict[dataset_name] = results_baseline
#
# logger.info("Evaluation results for {} in csv format:".format(dataset_name))
# print_csv_format(results_ours_dict)
# print_csv_format(results_baseline_dict)
if args.with_gambler:
with EventStorage(0) as storage:
with torch.no_grad():
# with open(os.path.join(args.output, "statistics_gambler_fg.csv"), "a") as my_csv:
# my_csv.write(f"img, "
# f"the highest score predicted, "
# f"gambler score before normalization, "
# f"gambler score after normalization, "
# f"Gt id, "
# f"predicted class id, "
# f"loss\n")
# with open(os.path.join(args.output, "statistics_gambler_highloss.csv"), "a") as my_csv:
# my_csv.write(f"img, "
# f"the highest score predicted, "
# f"gambler score before normalization, "
# f"gambler score after normalization, "
# f"Gt id, "
# f"predicted class id, "
# f"loss\n")
# with open(os.path.join(args.output, "statistics_baseline_fg.csv"), "a") as my_csv:
# my_csv.write(f"img, "
# f"the highest score predicted, "
# f"Gt id, "
# f"predicted class id, "
# f"loss\n")
# with open(os.path.join(args.output, "statistics_baseline_highloss.csv"), "a") as my_csv:
# my_csv.write(f"img, "
# f"the highest score predicted, "
# f"Gt id, "
# f"predicted class id, "
# f"loss\n")
with torch.no_grad():
with open(os.path.join(args.output, f"{args.dir_ours.split('/')[-2]}_one_image_allanchors.csv"), "a") as my_csv:
my_csv.write(f"img, "
f"the highest score predicted, "
f"gambler score before normalization, "
f"gambler score after normalization, "
f"Gt id, "
f"predicted class id, "
f"loss\n")
for img_indx, inputs in enumerate(train_data_loader):
# if img_indx > 2:
# break
input_images, generated_output, gt_classes, _, loss_dict = detector_ours(inputs)
mask = torch.ones_like(gt_classes)
gambler_loss_dict, weights, betting_map = gambler_ours(input_images,
generated_output['pred_class_logits'],
gt_classes,
mask,
detach_pred=False) # (N,AK,H,W)
gt_classes = gt_classes.flatten() #torch.Size([37632])
pred_class_logits = list_N_AK_H_W_to_NsumHWA_K(generated_output['pred_class_logits'], cfg.MODEL.RETINANET.NUM_CLASSES) #torch.Size([37632, 80])
# pred_class_sigmoids = (torch.sigmoid(pred_class_logits) - 0.5) * 256
pred_class_sigmoids = torch.sigmoid(pred_class_logits)
betting_map = list_N_AK_H_W_to_NsumHWA_K(betting_map, num_classes=1).flatten() # torch.Size([37632])
weights = weights.flatten() # torch.Size([37632])
loss_before_weighting = list_N_AK_H_W_to_NsumHWA_K(gambler_loss_dict['NAKHW_loss'], num_classes=1).flatten() # torch.Size([37632])
# foreground_idxs = (gt_classes >= 0) & (gt_classes != cfg.MODEL.RETINANET.NUM_CLASSES)
# filtered_anchor_ind = foreground_idxs
# print(f"number of filtered anchors to be displayed: {filtered_anchor_ind.sum()}")
#
# f_betting_map = betting_map[filtered_anchor_ind]
# f_weights = weights[filtered_anchor_ind]
# f_gt_classes = gt_classes[filtered_anchor_ind]
# f_pred_class_sigmoids = pred_class_sigmoids[filtered_anchor_ind, :]
# f_loss_before_weighting = loss_before_weighting[filtered_anchor_ind]
# for over all chosen anchors
for pred_class_sigmoid, gambler_score_bef, gambler_score_aft, matched_gt, loss in zip(pred_class_sigmoids, betting_map, weights, gt_classes, loss_before_weighting):
with open(os.path.join(args.output, f"{args.dir_ours.split('/')[-2]}_one_image_allanchors.csv"), "a") as my_csv:
my_csv.write(f"{inputs[0]['file_name'].split('/')[-1]}, "
f"{pred_class_sigmoid.max()}, "
f"{gambler_score_bef}, "
f"{gambler_score_aft}, "
f"{matched_gt}, "
f"{pred_class_sigmoid.argmax()}, "
f"{loss}\n")
# foreground_idxs = (gt_classes >= 0) & (gt_classes != cfg.MODEL.RETINANET.NUM_CLASSES)
# filtered_anchor_ind = (loss_before_weighting >= (0.1 * loss_before_weighting.max()))
# print(f"number of filtered anchors to be displayed: {filtered_anchor_ind.sum()}")
#
# f_betting_map = betting_map[filtered_anchor_ind]
# f_weights = weights[filtered_anchor_ind]
# f_gt_classes = gt_classes[filtered_anchor_ind]
# f_pred_class_sigmoids = pred_class_sigmoids[filtered_anchor_ind, :]
# f_loss_before_weighting = loss_before_weighting[filtered_anchor_ind]
#
# # for over all chosen anchors
# for pred_class_sigmoid, gambler_score_bef, gambler_score_aft, matched_gt, loss in zip(
# f_pred_class_sigmoids, f_betting_map, f_weights, f_gt_classes, f_loss_before_weighting):
# with open(os.path.join(args.output, "statistics_gambler_highloss.csv"), "a") as my_csv:
# my_csv.write(f"{inputs[0]['file_name'].split('/')[-1]}, "
# f"{pred_class_sigmoid.max()}, "
# f"{gambler_score_bef}, "
# f"{gambler_score_aft}, "
# f"{matched_gt}, "
# f"{pred_class_sigmoid.argmax()}, "
# f"{loss}\n")
##############################################################################################
# input_images, generated_output, gt_classes, loss_dict = detector_baseline(inputs)
#
# gt_classes = gt_classes.flatten() # torch.Size([37632])
# pred_class_logits = list_N_AK_H_W_to_NsumHWA_K(generated_output['pred_class_logits'],
# cfg.MODEL.RETINANET.NUM_CLASSES) # torch.Size([37632, 80])
# # pred_class_sigmoids = (torch.sigmoid(pred_class_logits) - 0.5) * 256
# pred_class_sigmoids = torch.sigmoid(pred_class_logits)
# # loss_before_weighting = list_N_AK_H_W_to_NsumHWA_K(gambler_loss_dict['NAKHW_loss'],
# # num_classes=1).flatten() # torch.Size([37632])
#
# foreground_idxs = (gt_classes >= 0) & (gt_classes != cfg.MODEL.RETINANET.NUM_CLASSES)
# filtered_anchor_ind = foreground_idxs
# print(f"number of filtered anchors to be displayed: {filtered_anchor_ind.sum()}")
#
# # f_betting_map = betting_map[filtered_anchor_ind]
# # f_weights = weights[filtered_anchor_ind]
# f_gt_classes = gt_classes[filtered_anchor_ind]
# f_pred_class_sigmoids = pred_class_sigmoids[filtered_anchor_ind, :]
# # f_loss_before_weighting = loss_before_weighting[filtered_anchor_ind]
#
# # for over all chosen anchors
# for pred_class_sigmoid, matched_gt in zip(f_pred_class_sigmoids, f_gt_classes):
# with open(os.path.join(args.output, "statistics_baseline_fg.csv"), "a") as my_csv:
# my_csv.write(f"{inputs[0]['file_name'].split('/')[-1]}, "
# f"{pred_class_sigmoid.max()}, "
# f"{matched_gt}, "
# f"{pred_class_sigmoid.argmax()}\n")
# # foreground_idxs = (gt_classes >= 0) & (gt_classes != cfg.MODEL.RETINANET.NUM_CLASSES)
# filtered_anchor_ind = (loss_before_weighting >= (0.1 * loss_before_weighting.max()))
# print(f"number of filtered anchors to be displayed: {filtered_anchor_ind.sum()}")
#
# f_betting_map = betting_map[filtered_anchor_ind]
# f_weights = weights[filtered_anchor_ind]
# f_gt_classes = gt_classes[filtered_anchor_ind]
# f_pred_class_sigmoids = pred_class_sigmoids[filtered_anchor_ind, :]
# f_loss_before_weighting = loss_before_weighting[filtered_anchor_ind]
#
# # for over all chosen anchors
# for pred_class_sigmoid, gambler_score_bef, gambler_score_aft, matched_gt, loss in zip(
# f_pred_class_sigmoids, f_betting_map, f_weights, f_gt_classes, f_loss_before_weighting):
# with open(os.path.join(args.output, "statistics_gambler_highloss.csv"), "a") as my_csv:
# my_csv.write(f"{inputs[0]['file_name'].split('/')[-1]}, "
# f"{pred_class_sigmoid.max()}, "
# f"{gambler_score_bef}, "
# f"{gambler_score_aft}, "
# f"{matched_gt}, "
# f"{pred_class_sigmoid.argmax()}, "
# f"{loss}\n")
break
if args.vis_bets:
from train_net import visualize_training_
with torch.no_grad():
with EventStorage(0) as storage:
for img_indx, inputs in enumerate(train_data_loader):
# if img_indx > 2:
# break
input_images, generated_output, gt_classes, _, loss_dict = detector_ours(inputs)
mask = torch.ones_like(gt_classes)
gambler_loss_dict, weights, betting_map = gambler_ours(input_images,
generated_output['pred_class_logits'],
gt_classes,
mask,
detach_pred=False) # (N,AK,H,W)
visualize_training_(gt_classes.clone().detach(), gambler_loss_dict["NAKHW_loss"],
weights, input_images, storage)
for _max in dmax:
if _max not in resultado[_type].keys():
resultado[_type][_max] = {'acerto':0,'erro':0}
_distances = [m.distance for m in match if m.distance < _max]
#print get_pos(t),get_pos(d), d, t
print len(_distances), check_hit(get_pos(t), get_pos(d)), get_pos(t), get_pos(d)
if len(_distances)> 5:
if check_hit(get_pos(t), get_pos(d)):
resultado[_type][_max]['acerto'] +=1
else:
resultado[_type][_max]['erro'] +=1
with open('json_data/resultado_01.json', 'w') as fp:
json.dump(resultado, fp)
#print resultado
# sys.exit()
#**********************************************************************************************
def save_to_json(fname, data):
with open(fname, 'w') as outfile:
json.dump(data, outfile)
def save(self): with open(Config.path, 'w') as f: json.dump(self, f, default=Config.config_to_dict)
async def inline_answer(_, inline_query: InlineQuery):
results = []
i_q = inline_query.query
string = i_q.lower()
str_x = i_q.split(" ", 2)
string_split = string.split()
if inline_query.from_user and inline_query.from_user.id == Config.OWNER_ID or inline_query.from_user.id in Config.SUDO_USERS:
MAIN_MENU = InlineQueryResultArticle(
title="Main Menu",
input_message_content=InputTextMessageContent(
" 𝐔𝐒𝐄𝐑𝐆𝐄-𝐗 𝗠𝗔𝗜𝗡 𝗠𝗘𝗡𝗨 "),
url="https://github.com/code-rgb/USERGE-X",
description="Userge-X Main Menu",
thumb_url="https://i.imgur.com/1xsOo9o.png",
reply_markup=InlineKeyboardMarkup(main_menu_buttons()))
results.append(MAIN_MENU)
if string == "syntax":
owner = [[
InlineKeyboardButton(text="Contact",
url="https://t.me/deleteduser420")
]]
results.append(
InlineQueryResultPhoto(
photo_url=
"https://coverfiles.alphacoders.com/123/123388.png",
caption="Hey I solved **𝚂𝚢𝚗𝚝𝚊𝚡's ░ Σrr♢r**",
reply_markup=InlineKeyboardMarkup(owner)))
if string == "rick":
rick = [[
InlineKeyboardButton(
text="🔍",
url="https://www.youtube.com/watch?v=dQw4w9WgXcQ")
]]
results.append(
InlineQueryResultArticle(
title="Not a Rick Roll",
input_message_content=InputTextMessageContent(
"Search Results"),
description="Definately Not a Rick Roll",
thumb_url="https://i.imgur.com/hRCaKAy.png",
reply_markup=InlineKeyboardMarkup(rick)))
if string == "alive":
random_alive = random.choice(ALIVE_IMGS)
buttons = [[
InlineKeyboardButton("🔧 SETTINGS",
callback_data="settings_btn"),
InlineKeyboardButton(text="⚡️ REPO",
url=Config.UPSTREAM_REPO)
]]
alive_info = f"""
**[USERGE-X](https://github.com/code-rgb/USERGE-X) is Up and Running**
• 🐍 Python : `v{versions.__python_version__}`
• 🔥 Pyrogram : `v{versions.__pyro_version__}`
• 🧬 𝑿 : `v{get_version()}`
🕔 Uptime : {userge.uptime}
"""
results.append(
InlineQueryResultPhoto(
photo_url=random_alive,
caption=alive_info,
reply_markup=InlineKeyboardMarkup(buttons)))
if string == "geass":
results.append(
InlineQueryResultAnimation(
animation_url="https://i.imgur.com/DeZHcRK.gif",
caption="To defeat evil, I must become a greater evil",
))
if string == "gapps":
buttons = [[
InlineKeyboardButton("Open GApps",
callback_data="open_gapps"),
InlineKeyboardButton("Flame GApps",
callback_data="flame_gapps")
],
[
InlineKeyboardButton("Nik GApps",
callback_data="nik_gapps")
]]
results.append(
InlineQueryResultArticle(
title="GApps",
input_message_content=InputTextMessageContent(
"[\u200c](https://i.imgur.com/BZBMrfn.jpg) **LATEST Android 10 arm64 GApps**"
),
description=
"Get Latest GApps Download Links Directly from SF",
thumb_url="https://i.imgur.com/Npzw8Ph.png",
reply_markup=InlineKeyboardMarkup(buttons)))
if len(string_split) == 2: #workaround for list index out of range
if string_split[0] == "ofox":
codename = string_split[1]
t = TelegraphPoster(use_api=True)
t.create_api_token('Userge-X')
photo = "https://i.imgur.com/582uaSk.png"
api_host = 'https://api.orangefox.download/v2/device/'
try:
cn = requests.get(f"{api_host}{codename}")
r = cn.json()
except ValueError:
return
s = requests.get(
f"{api_host}{codename}/releases/stable/last").json()
info = f"📱 **Device**: {r['fullname']}\n"
info += f"👤 **Maintainer**: {r['maintainer']['name']}\n\n"
recovery = f"🦊 <code>{s['file_name']}</code>\n"
recovery += f"📅 {s['date']}\n"
recovery += f"ℹ️ **Version:** {s['version']}\n"
recovery += f"📌 **Build Type:** {s['build_type']}\n"
recovery += f"🔰 **Size:** {s['size_human']}\n\n"
recovery += "📍 **Changelog:**\n"
recovery += f"<code>{s['changelog']}</code>\n\n"
msg = info
msg += recovery
notes_ = s.get('notes')
if notes_:
notes = t.post(title='READ Notes',
author="",
text=notes_)
buttons = [[
InlineKeyboardButton("🗒️ NOTES", url=notes['url']),
InlineKeyboardButton("⬇️ DOWNLOAD", url=s['url'])
]]
else:
buttons = [[
InlineKeyboardButton(text="⬇️ DOWNLOAD",
url=s['url'])
]]
results.append(
InlineQueryResultPhoto(
photo_url=photo,
thumb_url="https://i.imgur.com/o0onLYB.jpg",
title="Latest OFOX RECOVERY",
description=f"For device : {codename}",
caption=msg,
reply_markup=InlineKeyboardMarkup(buttons)))
if string == "repo":
results.append(REPO_X)
if str_x[0].lower() == "op" and len(str_x) > 1:
txt = i_q[3:] # TODO change it
opinion = os.path.join(PATH, "emoji_data.txt")
try:
view_data = json.load(open(opinion))
except:
view_data = False
if view_data:
# Uniquely identifies an inline message
new_id = {int(inline_query.id): [{}]}
view_data.update(new_id)
json.dump(view_data, open(opinion, 'w'))
else:
d = {int(inline_query.id): [{}]}
json.dump(d, open(opinion, 'w'))
buttons = [[
InlineKeyboardButton(
"👍", callback_data=f"op_y_{inline_query.id}"),
InlineKeyboardButton(
"👎", callback_data=f"op_n_{inline_query.id}")
]]
results.append(
InlineQueryResultArticle(
title="Ask For Opinion",
input_message_content=InputTextMessageContent(txt),
description="e.g @yourbot op Are Cats Cute?",
thumb_url="https://i.imgur.com/Zlc98qS.jpg",
reply_markup=InlineKeyboardMarkup(buttons)))
if string == "buttonnn":
async for data in BUTTON_BASE.find():
button_data = data['msg_data']
text, buttons = pb(button_data)
try:
photo_url = data['photo_url']
except KeyError:
photo_url = None
if photo_url:
results.append(
InlineQueryResultPhoto(photo_url=photo_url,
caption=text,
reply_markup=buttons))
else:
results.append(
InlineQueryResultArticle(
title=text,
input_message_content=InputTextMessageContent(
text),
reply_markup=buttons))
if str_x[0].lower() == "secret":
if len(str_x) == 3:
user_name = str_x[1]
msg = str_x[2]
try:
a = await userge.get_users(user_name)
user_id = a.id
except:
return
secret = os.path.join(PATH, "secret.txt")
try:
view_data = json.load(open(secret))
except:
view_data = False
if view_data:
# Uniquely identifies an inline message
new_id = {
str(inline_query.id): {
'user_id': user_id,
'msg': msg
}
}
view_data.update(new_id)
json.dump(view_data, open(secret, 'w'))
else:
d = {
str(inline_query.id): {
'user_id': user_id,
'msg': msg
}
}
json.dump(d, open(secret, 'w'))
buttons = [[
InlineKeyboardButton(
"🔐 REVEAL",
callback_data=f"secret_{inline_query.id}")
]]
results.append(
InlineQueryResultArticle(
title="Send A Secret Message",
input_message_content=InputTextMessageContent(
f"📩 <b>TOPSECRET!</b> for {user_name}. Only he/she can open it."
),
description=f"Send Secret Message to: {user_name}",
thumb_url="https://i.imgur.com/c5pZebC.png",
reply_markup=InlineKeyboardMarkup(buttons)))
else:
results = [(InlineQueryResultArticle(
title="Send A Secret Message",
input_message_content=InputTextMessageContent(
"Do `.secret` for more info"),
description="secret @username message ..."))]
await inline_query.answer(results=results,
cache_time=1,
switch_pm_text="🔒 SECRETS",
switch_pm_parameter="start")
return
else:
results.append(REPO_X)
try:
if not len(results) == 0:
await inline_query.answer(results=results, cache_time=1)
except MessageEmpty:
return
def persist_map(self): with open(self.map_file, 'w') as fh: json.dump(self.map, fh)
async def decode_records(records):
total = len(records)
print("Fetching details")
async with aiohttp.ClientSession() as session:
for i, r in enumerate(records):
print("({}/{})Processing {}".format(i + 1, total, r))
path = os.path.join(basedir, r)
with open(path) as f:
data = json.load(f)
if "data" in data:
print("({}/{})Data present in {}, skipping".format(i + 1, total, r))
continue
if "dataUrl" in data:
url = data["dataUrl"]
print("({}/{})Fetching details for {}".format(i + 1, total, r))
async with session.get(url) as res:
details = await res.read()
print("({}/{})Fetched details for {}".format(i + 1, total, r))
data["data"] = base64.b64encode(details).decode()
with open(path, "w") as f:
print("({}/{})Saving {}".format(i + 1, total, r))
json.dump(data, f, indent=2, ensure_ascii=False)
continue
print("({}/{})Neither data or dataUrl in {}, skipping".format(i + 1, total, r))
print("Decoding details")
for i, r in enumerate(records):
print("({}/{})Processing {}".format(i + 1, total, r))
path = os.path.join(basedir, r)
with open(path) as f:
data = json.load(f)
if "details" in data:
print("({}/{})Details present in {}, skipping".format(i + 1, total, r))
continue
blob = base64.b64decode(data['data'])
wrapper = pb.Wrapper()
wrapper.ParseFromString(blob)
assert wrapper.name == '.lq.GameDetailRecords'
records_data = wrapper.data
records = pb.GameDetailRecords()
records.ParseFromString(records_data)
records = records.records
results = []
for record in records:
wrapper = pb.Wrapper()
wrapper.ParseFromString(record)
name = wrapper.name.replace(".lq.", "")
cls = getattr(pb, name)
obj = cls()
obj.ParseFromString(wrapper.data)
result = MessageToDict(obj)
result['@type'] = name
results.append(result)
data["details"] = results
with open(path, "w") as f:
print("({}/{})Saving {}".format(i + 1, total, r))
json.dump(data, f, indent=2, ensure_ascii=False)
def saveConfig(self):
with open('config.json', 'w') as fp:
json.dump(self.config, fp, sort_keys=True, indent=4)
# if the key matches the hashtag print that part of the data
for hashTag in v:
# the word variable is processed as an entire string not a list
words = v[hashTag]
# taking the [] placheholders out as in this they do not signify a list type
words = words.replace('[','')
words = words.replace(']','')
hashtags.append([words])
if i == "date":
for date in v:
# find the current date
current_date = v[date]
dates.append(current_date)
# combine both of these list into one
combinedList = list(zip(dates,hashtags))
# sort that list by most recent date first
combinedList = sorted(combinedList, key = lambda x:x[0], reverse=True)
#Include the list into dictionary
data["HashtagsByDate"] = combinedList
#Append the updated column into original JSON file
with open('output.json','a') as file_df:
data = json.dump(data,file_df)
file_df.close()
def next_idiom_list(strx):
""" string to int list """
ret_list = list()
strxs = strx.split()
for i in range(len(data)):
stri = data[i]["english"]
stris = stri.split()
if strxs[-1] == stris[0]:
ret_list.append(i)
return ret_list
new_list = list()
for i in range(n):
if (i % 1000) == 0:
print(i, "idioms finished.")
idiom = data[i]
new_data = dict()
new_data["next"] = next_idiom_list(idiom["english"])
new_data["next_num"] = len(new_data["next"])
new_data["word"] = idiom["word"]
new_data["english"] = idiom["english"]
new_data["no"] = i
new_list.append(new_data)
# sorting
sorted_d = sorted(new_list, key=lambda x: x['next_num'])
with open('../data/idiom-graph.json', 'w+') as outfile:
json.dump(sorted_d, outfile)
def save_json(data, path):
with open(path, 'w') as f:
json.dump(data, f)
def clear():
with open("./json/task.json", 'w') as updatejson:
json.dump([], updatejson)
return {"Success": "Successfully Deleted"}, 200, {'Content-type': 'application/json'}
from jina.peapods.runtimes.asyncio.rest.app import get_fastapi_app
JINA_LOGO_URL = 'https://api.jina.ai/logo/logo-product/jina-core/horizontal-layout/colored/Product%20logo_Core_vertical_colorful%402x-margin.png'
DAEMON_SCHEMA_FILENAME = 'daemon.json'
GATEWAY_SCHEMA_FILENAME = 'gateway.json'
args = set_gateway_parser().parse_args([])
logger = JinaLogger('')
gateway_app = get_fastapi_app(args, logger)
gateway_schema = gateway_app.openapi()
gateway_schema['info']['x-logo'] = {'url': JINA_LOGO_URL}
gateway_schema['servers'] = []
gateway_schema['servers'].append({
'url': f'http://localhost:{args.port_expose}',
'description': 'Local Jina gateway'
})
with open(GATEWAY_SCHEMA_FILENAME, 'w') as f:
json.dump(gateway_schema, f)
args = get_main_parser().parse_args([])
daemon_app = _get_app()
daemon_schema = daemon_app.openapi()
daemon_schema['info']['x-logo'] = {'url': JINA_LOGO_URL}
daemon_schema['servers'] = []
daemon_schema['servers'].append({
'url': f'http://localhost:{args.port_expose}',
'description': 'Local JinaD server'
})
with open(DAEMON_SCHEMA_FILENAME, 'w') as f:
json.dump(daemon_schema, f)
from collections import namedtuple
import ast
import json
from pathlib import Path
for p in Path('nb201/').glob("*.log"):
archs = []
with open(p) as f:
for line in f:
if "Anytime results: " in line:
arch_eval = ast.literal_eval(line[line.index("{'cifar10-valid':"):].rstrip())
archs.append(arch_eval)
valid_acc = []
with open(p) as f:
for line in f:
if "Validation accuracy:" in line:
pos = line.index("Validation accuracy: ")
val = ast.literal_eval(line[pos + 21:pos + 29])
valid_acc.append(val)
with open('nb201/{}.json'.format(p.stem), 'w') as f:
json.dump({
'test_acc': archs,
'valid_acc': valid_acc}, f)
r = requests.get(url.format(**var))
j = json.loads(r.content.decode("utf-8"))
yield j['results']
movies = list(crawl_movies(total_pages))
movies = list(itertools.chain.from_iterable(movies))
return movies
movies = crawl_top_rated_movies_id()
# Save top_rated movies
data_file = os.path.join(data_folder, 'top_rated_movies.json')
with open(data_file, 'w') as stream:
json.dump(movies, stream)
# Load top_rated movies and transform to dictionary
data_file = os.path.join(data_folder, 'top_rated_movies.json')
with open(data_file, 'r') as stream:
movies_load = json.load(stream)
movies_load = dict([(d['id'], d) for d in movies_load])
def crawl_movies_detail(movie_id):
var = {
'movie_id': movie_id,
'api_key': config['api_key'],
'language': 'en-US',
}
