def write(self):
f = open(self.__fn(), 'w')
try:
json.dump(self.content, f, indent=2)
except:
pass
f.close()
def saveUserFlow(userFlow):
path = xbmc.translatePath(MDB_ADDON.getAddonInfo('profile'))
if not os.path.exists(path):
try:
os.makedirs(path)
except OSError:
print "unable to create directory for saving userflow; userflow will not be saveds"
return # ignore
try:
file = os.path.join(path, '%s.json' % ADDON.getAddonInfo('id'))
# remove entries older than 24 hours
# we compare strings rather the datetimes (a little hackish though)
# but datetime.datetime.strptime() often fail for no apparent reason
# see http://forum.xbmc.org/showthread.php?tid=112916
oneDayAgo = datetime.datetime.now() - datetime.timedelta(days=1)
oneDayAgoStr = oneDayAgo.isoformat()
for dateStr in userFlow.keys():
if dateStr < oneDayAgoStr:
del userFlow[dateStr]
simplejson.dump(userFlow, open(file, 'w'))
except Exception:
print "problem saving userflow json file"
def stream_for_day(self, day):
cache_filename = self.cache_filename_base(day) + '.json.gz'
if os.path.isfile(cache_filename):
log.info('found stream in cache: %r on %s', self.keyword, day)
with gzip.open(cache_filename, 'rb') as f:
for mention in json.load(f):
yield mention
else:
cache_file_dirname = os.path.dirname(cache_filename)
if not os.path.isdir(cache_file_dirname):
os.makedirs(cache_file_dirname)
tmp_cache_filename = (cache_filename +
'.%05d.tmp' % randint(0, 10000))
with gzip.open(tmp_cache_filename, 'wb') as f:
f.write('[')
first = True
stream = super(CachingMentionCounter, self).stream_for_day(day)
for mention in stream:
if not first:
f.write(', ')
json.dump(mention, f)
yield mention
f.write(']')
os.rename(tmp_cache_filename, cache_filename)
log.info('cached stream for %r on %s', self.keyword, day)
def update_notified(jobs):
with open(filename, 'r') as f:
data = simplejson.load(f)
for job in jobs:
data['jobs'][job]['notified'] = True
with open(filename, 'w') as f:
simplejson.dump(data, f)
def sync(self):
"""Synchronise and update the stored state to the in-memory state."""
if self.filepath:
with open(self.filepath, "w") as serialised_file:
simplejson.dump(self.state, serialised_file)
else:
print "Filepath to the persistence file is not set. State cannot be synced to disc."
def setUp(self):
self.testfolder = "foo-test"
rm_rf("/tmp/vpm")
rm_rf(self.testfolder)
ctrl = ControlFile()
ctrl.settings["Name"] = "foo"
ctrl.settings["Version"] = "2.0.0"
ctrl.settings["Type"] = "Cartridge"
ctrl.settings["Provides"] = "foobar (3.0.0)"
ctrl.settings["Bundles"] = "foobar"
make_vpm(self.testfolder, ctrl)
filemodes = [{"path": "package-file", "mode": "0755"}]
file = os.path.join(self.testfolder, VPM.META_DIR_NAME, VPM.FILEMODE_FILE_NAME)
fp = open(file, "w")
simplejson.dump(filemodes, fp)
fp.close()
self.env = VPM.Environment("/tmp/vpm")
self.p = VPM.Package(self.env)
r, self.v, e = self.p.pack(self.testfolder, ".")
self.vpmfile = self.v
def run(self, options, **kwargs):
config_file = os.path.join(options.app_path, "wsgid.json")
f = self._open_config_file(config_file)
s = f.read()
cfg_values = {}
if s:
cfg_values = simplejson.loads(s)
# Copy the values
self._override_if_not_none("wsgi_app", cfg_values, options.wsgi_app)
self._override_if_not_none("debug", cfg_values, options.debug)
if options.workers > 1:
self._override_if_not_none("workers", cfg_values, options.workers, convert_func=int)
self._override_if_not_none("keep_alive", cfg_values, options.keep_alive)
self._override_if_not_none("chroot", cfg_values, options.chroot)
self._override_if_not_none("no_daemon", cfg_values, options.no_daemon)
self._override_if_not_none("recv", cfg_values, options.recv)
self._override_if_not_none("send", cfg_values, options.send)
self._override_if_not_none("mongrel2_chroot", cfg_values, options.mongrel2_chroot)
# Custom config command options
if options.no_debug:
cfg_values["debug"] = str((not options.no_debug))
# Rewrite the config file
f.seek(0)
f.truncate()
simplejson.dump(cfg_values, f, indent=" ")
f.close()
def get_cached_categories(self, parent_id):
categories = None
fpath = os.path.join(self.plugin.get_cache_dir(), 'canada.on.demand.%s.categories.cache' % (self.get_cache_key(),))
try:
if os.path.exists(fpath):
data = simplejson.load(open(fpath))
if data['cached_at'] + self.category_cache_timeout >= time.time():
logging.debug("using cached Categories")
categories = data['categories']
except:
logging.debug("no cached Categories path")
if not categories:
logging.debug('http-retrieving categories')
url = self.get_categories_json(parent_id)
logging.debug('get_cached_categories(p_id=%s) url=%s'%(parent_id, url))
categories = self.parse_callback(self.plugin.fetch(url, self.cache_timeout).read())['items']
if self.category_cache_timeout > 0:
fpath = os.path.join(self.plugin.get_cache_dir(), 'canada.on.demand.%s.categories.cache' % (self.short_name,))
fh = open(fpath, 'w')
simplejson.dump({'cached_at': time.time(), 'categories': categories}, fh)
fh.close()
return categories
def test_tuple_array_dump(self):
t = (1, 2, 3)
expect = json.dumps(list(t))
# Default is True
sio = StringIO()
json.dump(t, sio)
self.assertEqual(expect, sio.getvalue())
sio = StringIO()
json.dump(t, sio, tuple_as_array=True)
self.assertEqual(expect, sio.getvalue())
self.assertRaises(TypeError, json.dump, t, StringIO(),
tuple_as_array=False)
# Ensure that the "default" does not get called
sio = StringIO()
json.dump(t, sio, default=repr)
self.assertEqual(expect, sio.getvalue())
sio = StringIO()
json.dump(t, sio, tuple_as_array=True, default=repr)
self.assertEqual(expect, sio.getvalue())
# Ensure that the "default" gets called
sio = StringIO()
json.dump(t, sio, tuple_as_array=False, default=repr)
self.assertEqual(
json.dumps(repr(t)),
sio.getvalue())
def main():
"""Generates and outputs (as json file) the data for the main dropdown menus on the home page"""
parse_db_uri(SQL_URI)
species = get_species()
groups = get_groups(species)
types = get_types(groups)
datasets = get_datasets(types)
species.append(('All Species', 'All Species'))
groups['All Species'] = [('All Groups', 'All Groups')]
types['All Species'] = {}
types['All Species']['All Groups'] = [('Phenotypes', 'Phenotypes')]
datasets['All Species'] = {}
datasets['All Species']['All Groups'] = {}
datasets['All Species']['All Groups']['Phenotypes'] = [('All Phenotypes','All Phenotypes')]
data = dict(species=species,
groups=groups,
types=types,
datasets=datasets,
)
#print("data:", data)
output_file = """./wqflask/static/new/javascript/dataset_menu_structure.json"""
with open(output_file, 'w') as fh:
json.dump(data, fh, indent=" ", sort_keys=True)
def generate_top():
from collections import defaultdict
import simplejson as json
import operator
from_product = defaultdict(lambda: 0)
results = defaultdict(lambda: 0)
for product in cols['product'].find().sort('_id', -1):
for k in product.keys():
from_product[k] += 1
product_keys = dict(from_product)
for w in list(product_keys.keys()):
jieba.add_word(w, tag='nz')
progress = 0
for comment in cols['mobile_comment'].find(projection={'content': 1}):
c = comment['content']
words = jieba.analyse.extract_tags(c, topK=20, withWeight=False, allowPOS=('ns', 'n', 'nz'))
for w in words:
results[w] += 1
progress += 1
if progress % 100 == 0:
print('Current Progress: ', progress)
sorted_x = reversed(sorted(dict(results).items(), key=operator.itemgetter(1)))
json.dump(
list(sorted_x),
open('sorted_mobile.json', mode='w', encoding='utf-8'),
ensure_ascii=False,
indent=2
)
def putJsonIntoFile(fname, jdata):
if fname and isinstance(jdata, dict):
path = os.path.join(os.getcwd(), fname)
f = open(path, 'w+')
jdata['time'] = int(time.time())
json.dump(jdata, f)
f.close()
def main(argv):
data, lang = argv[-2:]
f = myStore.load(data)
lang = f.newSymbol(lang)
it = {
"rules": asGrammar(f, lang),
"tokens": tokens(f, lang),
"first": sets(f, lang, EBNF.first),
"follow": sets(f, lang, EBNF.follow),
}
if "--pprint" in argv:
from pprint import pprint
pprint(it)
elif "--yacc" in argv:
toYacc(it)
elif "--ply" in argv:
toPly(it)
else:
import simplejson # http://cheeseshop.python.org/pypi/simplejson
import sys
start = it["rules"][0][0]
print "SYNTAX_%s = " % start,
simplejson.dump(it, sys.stdout)
def build_tooltool_manifest():
def key_sort(item):
item = item[0]
if item == 'size':
return 0
if item == 'digest':
return 1
if item == 'algorithm':
return 3
return 4
basedir = os.path.split(os.path.realpath(sys.argv[0]))[0]
tooltool = basedir + '/tooltool.py'
setup = basedir + '/setup.sh'
manifest = 'clang.manifest'
check_run(['python', tooltool, '-m', manifest, 'add',
setup, 'clang.tar.bz2'])
data = simplejson.load(file(manifest))
data = [{'clang_version' : 'r%s' % llvm_revision }] + data
out = file(manifest,'w')
simplejson.dump(data, out, indent=0, item_sort_key=key_sort)
out.write('\n')
assert data[2]['filename'] == 'clang.tar.bz2'
os.rename('clang.tar.bz2', data[2]['digest'])
def editfile(fn, password):
filetype = aespckfile
if ".json" in fn:
filetype = aesjsonfile
db = filetype.load(fn, password)
f = tempfile.NamedTemporaryFile()
json.dump(db, f, indent=2)
f.flush()
mtime = os.path.getmtime(f.name)
while True:
subprocess.call([os.getenv("EDITOR") or "editor", f.name])
if os.path.getmtime(f.name) == mtime:
print "Not updated"
break
try:
f.seek(0)
db = json.load(f)
filetype.dump(fn, db, password)
break
except Exception, e:
print "Error in json"
print e
print "Try again (y/n)? ",
input = raw_input()
if not input.lower().startswith("y"):
break
def export(filename):
questionList = []
for question in __getQuestionsCollection().find():
questionList.append(question)
fd = open(filename, 'w')
simplejson.dump(questionList, fd, indent=True)
fd.close()
def update(self):
"""
"""
params = {'key': yeti_config.get('threattracking', 'google_api_key')}
# , 'includeGridData': 'True'} - we don't want to do that. 200Mo file.
base = "https://sheets.googleapis.com/v4/spreadsheets/" + yeti_config.get(
'threattracking', 'sheet_key')
self.api = hammock.Hammock(base, params=params)
if False:
r = self.api.GET()
if r.status_code != 200:
raise requests.ConnectionError(
'Return code for {query} is {code}'.format(
query=r.request.url, code=r.status_code))
sheets = r.json()['sheets']
json.dump(sheets, open("actor.sheets.json", "w"))
else:
sheets = json.load(open("actor.sheets.json", "r"))
# print(pprint.pformat(sheets))
for s_p in sheets:
s = s_p['properties']
title = s['title']
if title in ['Home', '_Malware', '_Download', '_Schemes',
'_Sources']:
continue
size = s['gridProperties']
# print(title, size['columnCount'], size['rowCount'])
actors_list_info = self.each_sheet_work(s)
self.create_entities(title, actors_list_info)
return
def processmultimsg(mmsg, callback):
mmsgobj = MultiMsg(**json.loads(mmsg))
fname = "multimsg.dat"
if os.path.exists(fname):
mmsgfile = open(fname, 'r')
mmsgdict = json.load(mmsgfile)
mmsgfile.close()
else:
mmsgdict = {}
msginfo = None
if mmsgobj.hash in mmsgdict:
msglist = mmsgdict[mmsgobj.hash]
for i in range( len(msglist) ):
msglist[i] = MultiMsg(**msglist[i])
msglist.append(mmsgobj)
if len(msglist) == mmsgobj.total:
origmsg = reconstructmsg(msglist)
msginfo = callback(origmsg)
del(mmsgdict[mmsgobj.hash])
else:
mmsgdict[mmsgobj.hash] = [mmsgobj]
mmsgfile = open(fname, 'w')
json.dump(mmsgdict, mmsgfile)
mmsgfile.close()
return msginfo
def sign_up(request):
if request.COOKIES.get('member'):
email = request.COOKIES.get('member')
response = render(request, 'index.html',{'member':'logout '+ email[0:email.find('@')]})
response.set_cookie("member",email)
return response
if request.method == 'POST':
email = request.POST.get('email')
passwd = request.POST.get('passwd')
passwd = sha512(passwd).hexdigest()
temp_dict = {email:passwd}
in_file = open('member.json')
memberlist = simplejson.load(in_file)
memberlist.update(temp_dict)
in_file.close()
out_file = open('member.json','w')
simplejson.dump(memberlist,out_file, indent=4)
out_file.close()
response = render_to_response('index.html',{'member':'logout '+ email[0:email.find('@')]},
context_instance=RequestContext(request))
response.set_cookie("member",email)
return response
def write(self):
path = os.path.join(os.path.dirname(__file__), '..',
'ionize', 'Database', 'ion_data.json')
print(path)
with open(path, 'w') as ion_file:
json.dump(self.data, ion_file,
sort_keys=True, indent=4, separators=(',', ': '))
def to_zipfile(obj, zipfile, path=''):
for key in obj.keys():
new_path = join_path(path, key)
if isinstance(obj[key], h5py.Group):
to_zipfile(obj[key], zipfile, new_path)
else:
fname = join_path(path, key+'.dat')
# some members are empty... skip them.
if sum(obj[key].shape) == 0:
continue
value = obj[key].value
formats = {
'S': '%s',
'f': '%.8g',
'i': '%d',
'u': '%d' }
if value.dtype.kind in formats:
fd, fn = tempfile.mkstemp()
os.close(fd) # to be opened by name
if DEBUG: print fname, value.dtype.kind
if len(value.shape) > 2:
with open(fn, 'w') as f:
simplejson.dump(value.tolist(), f)
else:
savetxt(fn, value, delimiter='\t', fmt=formats[value.dtype.kind])
zipfile.write(fn, fname)
os.remove(fn)
else:
print "unknown type of array: ", fname, value.dtype
def updateDagNodes(self, dagnodes, debug=False):
"""
Update the networkx nodes and links attributes from scene values.
:param dagnodes: list of dag node objects.
:type dagnodes: list
"""
if type(dagnodes) not in [list, tuple]:
dagnodes=[dagnodes,]
for dag in dagnodes:
log.debug('Graph: updating dag node "%s"' % dag.name)
nid = dag.id
if nid in self.network.nodes():
#self.network.node[nid].update(dag.data)
dag_data = json.loads(str(dag), object_pairs_hook=dict)
nx_data = self.network.node[nid]
nx_data.update(dag_data)
if debug:
# write temp file
filename = os.path.join(os.path.dirname(self.autosave_path), '%s.json' % dag.name)
fn = open(filename, 'w')
json.dump(dag_data, fn, indent=4)
fn.close()
def outputDashboard(self):
log.debug("Creating dashboard")
dirname = self.config.get('main', 'dashboard_dir')
if not os.path.exists(dirname):
# Copy in the rest of html
shutil.copytree('html/dashboard', dirname)
shutil.copytree('html/flot', '%s/flot' % dirname)
shutil.copytree('html/jquery', '%s/jquery' % dirname)
filename = os.path.join(dirname, 'testdata.js')
fp = open(filename + ".tmp", "w")
now = time.asctime()
fp.write("// Generated at %s\n" % now)
fp.write("gFetchTime = ")
json.dump(now, fp, separators=(',',':'))
fp.write(";\n")
fp.write("var gData = ")
# Hackity hack
# Don't pretend we have double precision here
# 8 digits of precision is plenty
try:
json.encoder.FLOAT_REPR = lambda f: "%.8g" % f
except:
pass
json.dump(self.dashboard_data, fp, separators=(',',':'), sort_keys=True)
try:
json.encoder.FLOAT_REPR = repr
except:
pass
fp.write(";\n")
fp.close()
os.rename(filename + ".tmp", filename)
def main(inputPtnPath,outputPath):
start_time = datetime.now()
model, table = projizz.readPrefixTreeModelWithTable("./yagoPatternTree.model","./yagoPatternTree.table")
properties = projizz.buildYagoProperties({})
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
t = 0
result = []
for filename in os.listdir(inputPtnPath):
if ".json" in filename:
result.append(pool.apply_async(filterFunction, (t,filename,inputPtnPath,model,table,copy.deepcopy(properties) )))
t += 1
pool.close()
pool.join()
for res in result:
r = res.get()
for rela in r:
for ptnId in r[rela]:
if not ptnId in properties[rela]:
properties[rela][ptnId] = {"total":0,"support":0}
properties[rela][ptnId]["total"] += r[rela][ptnId]["total"]
properties[rela][ptnId]["support"] += r[rela][ptnId]["support"]
json.dump(properties,open(outputPath,"w"))
diff = datetime.now() - start_time
print "Spend %d.%d seconds" % (diff.seconds, diff.microseconds)
def generateShowsAndSave(self):
f = open('../../xml/tv3shows.json', 'w')
for show in self.getMainMenu():
# Load and read the URL
f2 = urllib2.urlopen(EPISODE_URL % (show['url']))
text = f2.read()
f2.close()
key = show['Title']
try:
showkeys = self.KNOWN_TV3_SHOWS[key].keys()
print 'Updating ' + show['Title']
self.KNOWN_TV3_SHOWS[key]['']
except:
print 'Adding ' + show['Title']
self.KNOWN_TV3_SHOWS[key] = {}
self.KNOWN_TV3_SHOWS[key]['Title'] = show['Title']
REGEXP = '<div id="content" style="background-image: url\((.*?)\)">'
for mymatch in re.findall(REGEXP, text, re.MULTILINE):
fanart = mymatch
self.KNOWN_TV3_SHOWS[key]['Fanart_Image'] = fanart
S.dump(self.KNOWN_TV3_SHOWS, f, indent=4)
f.close()
def display_matching_trips(request, trip_id=None, lib=None):
"""Make a request to the BV server to find matching trips. Format the
output to be read by javascript clientside code.
"""
def to_json(trip):
return [get_trip_dict(t) for t in trips]
trip_search_type = int(request.POST['trip_type'])
results = lib.search_trip(trip_id=trip_id, **unicode_to_dict(request.POST))
trip_demands = results['trip_demands']
trip_offers = results['trip_offers']
trip = results['trip']
if trip_search_type == TRIP_OFFER:
trips = trip_demands
else:
trips = trip_offers
response_dict = {
'authenticated': is_bvoauth_authenticated(request),
}
if not trip_id:
response_dict['trips'] = to_json(trips)
else:
response_dict['trip_demands'] = to_json(trip_demands)
response_dict['trip_offers'] = to_json(trip_offers)
resp = HttpResponse()
simplejson.dump(response_dict , resp, ensure_ascii=False, separators=(',',':'))
return resp
def main(inputPath,outputFileName):
properties = buildProperties("../naive_model/PbR/")
start_time = datetime.now()
result = []
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
t = 0
for filename in os.listdir(inputPath):
if ".json" in filename:
partAns = copy.deepcopy(properties)
result.append(pool.apply_async(findAnwser, (t,filename,inputPath,partAns, )))
t += 1
pool.close()
pool.join()
for res in result:
r = res.get()
for m in r:
properties[m]["tp"] += r[m]["tp"]
properties[m]["tn"] += r[m]["tn"]
properties[m]["fp"] += r[m]["fp"]
properties[m]["fn"] += r[m]["fn"]
print "start write out to %s" % (outputFileName)
json.dump(properties,open(outputFileName,"w"))
diff = datetime.now() - start_time
print "Spend %d.%d seconds" % (diff.seconds, diff.microseconds)
def write(self, filename, auto=False, data={}):
"""
Write the graph to scene file.
:param str filename: file to save.
:param bool auto: file is an autosave, don't set it as the current scene.
:param dict data: dictionary of graph data.
:returns: current scene name.
:rtype: str
"""
# callbacks
self.graphAboutToBeSaved()
if not data:
data = self.snapshot()
fn = open(filename, 'w')
json.dump(data, fn, indent=4)
fn.close()
if auto:
self._autosave_file = filename
# don't set the current autosave filename as the scene, use the parent filename
filename = filename[:-1]
# callbacks
self.graphSaved()
return self.setScene(filename)
def savemsgstore():
try:
f = open('generalmessage.json','w')
json.dump(generalmessagestore,f)
f.close()
except:
pass
def gen_data(self, fname):
"""
:fname : input file, every line means a single data
:rtype : List[List[float]]: data matrix
"""
lines = [ self.langConvHandler.convert(line.strip().lower()) for line in codecs.open(fname, "rb","utf-8") if len(line) > 6]
# lines = list(set(lines)) # remove duplicates
logging.info("number of data %d " % len(lines))
cut_lines = [" ".join(jieba.cut(line)) for line in lines]
# transform to tfidfVec
tfidfVec = TfidfVectorizer(max_features = 3000)
tfidf_data = tfidfVec.fit_transform(cut_lines)
tfidf_data = tfidf_data.toarray()
# save origin text
with open("./output/origin_lines.txt", "wb") as fw:
json.dump(lines, fw)
# save vectorize data
np.save("./output/tfidf.corpus.npy", tfidf_data)
self.lines = lines
self.tfidf_data = tfidf_data
article['source'] = source
data += [article]
return {'data': data, 'next': next_url}
def scrape_page(FACEBOOK_PAGE_ID, ACCESS_TOKEN):
URL = 'https://graph.facebook.com/v2.12/' + FACEBOOK_PAGE_ID +\
'?fields=posts.limit(100)' \
'{id,created_time,message,attachments,link,permalink_url,shares,%20status_type,%20comments.limit(1000).summary(true),reactions.type(LIKE).summary(total_count).as(like),reactions.type(LOVE).summary(total_count).as(love),reactions.type(HAHA).summary(total_count).as(haha),reactions.type(WOW).summary(total_count).as(wow),reactions.type(SAD).summary(total_count).as(sad),reactions.type(ANGRY).summary(total_count).as(angry),full_picture,picture}&access_token=' + ACCESS_TOKEN + '&pretty=0;'
data = read_posts(URL, FACEBOOK_PAGE_ID)
print(data)
articles = data['data']
next_link = data['next']
while next_link != '':
data = read_posts(next_link, FACEBOOK_PAGE_ID)
articles = articles + data['data']
next_link = data['next']
return articles
if __name__ == '__main__':
output = scrape_page("welovekarylle", config.ACCESS_TOKEN)
with open("output_fb.json", 'w') as f:
json.dump(output, f)
else:
mainAnswer['edited by'] = "N/A"
mainAnswer['answered by'] = initialOutputAD[anscount]
if (anscount != 0):
initialOutput.append(mainAnswer)
anscount = anscount + 1
myStr = initialOutputAD[0]['Date']
if "edited" in myStr:
finalMainAnswer['questions'] = {
"title": soup.title.text,
"Body": htmlpage_text[0],
"Upvote": vote_container[0],
"Edited by": initialOutputAD[0],
"Asked By": initialOutputAD[1]
}
else:
finalMainAnswer['questions'] = {
"title": soup.title.text,
"Body": htmlpage_text[0],
"Upvote": vote_container[0],
"Edited by": "N/A",
"Asked By": initialOutputAD[0]
}
finalMainAnswer['answers'] = initialOutput
with open("Nodejs.json", "w") as newFile:
json.dump(finalMainAnswer, newFile, indent=4)
print("Success!")
else:
print("Can't handle URL")
def dump(obj, fp, *args, **kwargs):
if is_simplejson:
kwargs['namedtuple_as_object'] = False
return json.dump(obj, fp, *args, **kwargs)
def _saveRegistrationParams(self):
""" Save registration parameters (app_id/token) to a local file """
log(">>> _saveRegistrationParams")
with open(self.registrationSaveFile, 'w') as outfile:
json.dump(self.registration, outfile)
def fetch_participants(self):
identifier_name_pairs: List = []
for identifier, participant in self.__participants.items():
identifier_name_pairs.append({'identifier': identifier, 'nickname': participant.nickname})
return json.dump(identifier_name_pairs)
def save_dataset(dataset, path):
with open(path, 'w') as f:
json.dump(dataset, f)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import simplejson
if __name__ == '__main__':
data = {}
elements = [
'', 'サラマンダー', 'ウンディーネ', 'シルフ', 'ノーム', 'フレイミーズ', 'アクアンズ', 'エアロス',
'アーシーズ'
]
a = {
'↓': -1,
'↑1': 1,
'↑2': 2,
}
for line in open('element-up.txt'):
text = [x.strip() for x in line.split('\t')]
results = {}
for i in range(1, len(elements)):
results[elements[i]] = a[text[i]]
data[text[0]] = results
simplejson.dump(data,
open('element-up.json', 'w'),
indent=2,
ensure_ascii=False)
def dump(obj, fp, *args, **kwargs):
return json.dump(obj, fp, *args, **kwargs)
opter.exit(1)
else:
# Config loaded, update optparser's defaults and reparse
defaults.update(loadedConfig)
opter = cliarg_parser.getCommandlineParser(defaults)
opts, args = opter.parse_args()
# Save config argument
if opts.saveconfig is not None:
p("Saving config: %s" % (opts.saveconfig))
configToSave = dict(opts.__dict__)
del configToSave['saveconfig']
del configToSave['loadconfig']
del configToSave['showconfig']
json.dump(configToSave,
open(opts.saveconfig, "w+"),
sort_keys=True,
indent=4)
opter.exit(0)
# Show config argument
if opts.showconfig:
for k, v in opts.__dict__.items():
p(k, "=", str(v))
return
# Process values
if opts.batch:
opts.select_first = True
opts.always_rename = True
def create_archive_input(staging,
execution,
dep_extractor,
collect_tmpdir=False):
"""
Generates .gen.json and .isolate files corresponding to the
test 'execution', which must be a TestExecution instance.
The outputs are placed in the specified staging directory.
"""
argv = execution.argv
if not argv[0].endswith('run-test.sh') or len(argv) < 2:
logging.warning("Unable to handle test: %s", argv)
return
abs_test_exe = os.path.realpath(argv[1])
rel_test_exe = abs_to_rel(abs_test_exe, staging)
argv[1] = rel_test_exe
files = []
files.append(rel_test_exe)
deps = dep_extractor.extract_deps(abs_test_exe)
deps.extend(get_base_deps(dep_extractor))
# Deduplicate dependencies included via DEPS_FOR_ALL.
for d in set(deps):
# System libraries will end up being relative paths out
# of the build tree. We need to copy those into the build
# tree somewhere.
if is_outside_of_tree(d):
d = copy_system_library(d)
files.append(abs_to_rel(d, staging))
# Add data file dependencies.
if 'KUDU_DATA_FILES' in execution.env:
for data_file in execution.env['KUDU_DATA_FILES'].split(","):
# Paths are relative to the test binary.
path = os.path.join(os.path.dirname(abs_test_exe), data_file)
files.append(abs_to_rel(path, staging))
out_archive = os.path.join(staging.dir,
'%s.gen.json' % (execution.test_name))
out_isolate = os.path.join(staging.dir,
'%s.isolate' % (execution.test_name))
command = ['../../build-support/run_dist_test.py',
'-e', 'KUDU_TEST_TIMEOUT=%d' % (TEST_TIMEOUT_SECS - 30),
'-e', 'KUDU_ALLOW_SLOW_TESTS=%s' % os.environ.get('KUDU_ALLOW_SLOW_TESTS', 1),
'-e', 'KUDU_COMPRESS_TEST_OUTPUT=%s' % \
os.environ.get('KUDU_COMPRESS_TEST_OUTPUT', 0)]
for k, v in execution.env.iteritems():
if k == 'KUDU_TEST_TIMEOUT':
# Currently we don't respect the test timeouts specified in ctest, since
# we want to make sure that the dist-test task timeout and the
# underlying test timeout are coordinated.
continue
command.extend(['-e', '%s=%s' % (k, v)])
if collect_tmpdir:
command += ["--collect-tmpdir"]
command.append('--')
command += argv[1:]
archive_json = dict(args=["-i", out_isolate, "-s", out_isolate + "d"],
dir=rel_to_abs("."),
version=1)
isolate_dict = dict(variables=dict(command=command, files=files))
with open(out_archive, "w") as f:
json.dump(archive_json, f)
with open(out_isolate, "w") as f:
pprint.pprint(isolate_dict, f)
def combine(product_file, spl_extract, rx_extract, unii_extract, json_out):
json_output_file = open(json_out, 'w')
product = csv.DictReader(product_file, delimiter='\t')
label = read_json_file(open(spl_extract))
rx = read_json_file(open(rx_extract))
unii = read_json_file(open(unii_extract))
all_products = []
for row in product:
clean_product = {}
#the productid has a date on the front of it, so need to split
clean_product['id'] = row['PRODUCTID'].split('_')[1]
clean_product['application_number'] = row['APPLICATIONNUMBER']
clean_product['product_type'] = row['PRODUCTTYPENAME']
clean_product['generic_name'] = row['NONPROPRIETARYNAME']
clean_product['manufacturer_name'] = row['LABELERNAME']
#TODO(hansnelsen): add suffix as a separate field in output
clean_product['brand_name'] = (row['PROPRIETARYNAME'] + ' ' +
row['PROPRIETARYNAMESUFFIX'])
clean_product['product_ndc'] = row['PRODUCTNDC']
clean_product['dosage_form'] = row['DOSAGEFORMNAME']
clean_product['route'] = row['ROUTENAME']
clean_product['substance_name'] = row['SUBSTANCENAME']
all_products.append(clean_product)
all_labels = {}
for spl_data in label:
clean_label = {}
clean_label['spl_set_id'] = spl_data['spl_set_id']
clean_label['id'] = spl_data['id']
clean_label['spl_version'] = spl_data['spl_version']
clean_label['is_original_packager'] = spl_data['is_original_packager']
clean_label['spl_product_ndc'] = spl_data['ProductNDCs']
clean_label['original_packager_product_ndc'] = \
spl_data['OriginalPackagerProductNDSs']
clean_label['package_ndc'] = spl_data['PackageNDCs']
all_labels[clean_label['id']] = clean_label
all_uniis = {}
for unii_data in unii:
hid = unii_data['spl_id']
all_uniis[hid] = {'unii_indexing': unii_data['unii_indexing']}
combo = []
for product in all_products:
p = {}
product_id = product['id']
if product_id in all_labels:
label_with_unii = dict(product.items() +
all_labels[product_id].items())
if product_id in all_uniis:
p = dict(label_with_unii.items() +
all_uniis[product_id].items())
else:
p = dict(label_with_unii.items() +
{'unii_indexing': []}.items())
combo.append(p)
combo2 = []
rx_dict = {}
for this_rx in rx:
xid = this_rx['spl_set_id'] + ':' + this_rx['spl_version']
rx_dict[xid] = this_rx
combo_dict = {}
for row in combo:
cid = row['spl_set_id'] + ':' + row['spl_version']
if cid in combo_dict:
combo_dict[cid].append(row)
else:
combo_dict[cid] = [row]
for combo_key, combo_value in combo_dict.iteritems():
for pivot in combo_value:
tmp_dict = {}
if combo_key in rx_dict:
tmp_dict = dict(pivot.items() + rx_dict[combo_key].items())
else:
tmp_dict = dict(pivot.items() + {'rxnorm': []}.items())
combo2.append(tmp_dict)
for row in combo2:
json.dump(row, json_output_file, encoding='utf-8-sig')
json_output_file.write('\n')
def main():
parser = argparse.ArgumentParser(description="Generate synth_tiles.json")
parser.add_argument(
'--db_root', required=True)
parser.add_argument(
'--roi', required=True)
parser.add_argument(
'--synth_tiles', required=False)
args = parser.parse_args()
db = prjxray.db.Database(args.db_root)
g = db.grid()
synth_tiles = {}
synth_tiles['tiles'] = {}
with open(args.roi) as f:
j = json.load(f)
roi = Roi(
db=db,
x1=j['info']['GRID_X_MIN'],
y1=j['info']['GRID_Y_MIN'],
x2=j['info']['GRID_X_MAX'],
y2=j['info']['GRID_Y_MAX'],
)
synth_tiles['info'] = j['info']
for port in j['ports']:
if port['name'].startswith('dout['):
port_type = 'input'
is_clock = False
elif port['name'].startswith('din['):
is_clock = False
port_type = 'output'
elif port['name'].startswith('clk'):
port_type = 'output'
is_clock = True
else:
assert False, port
tile, wire = port['wire'].split('/')
# Make sure connecting wire is not in ROI!
loc = g.loc_of_tilename(tile)
if roi.tile_in_roi(loc):
# Or if in the ROI, make sure it has no sites.
gridinfo = g.gridinfo_at_tilename(tile)
assert len(db.get_tile_type(gridinfo.tile_type).get_sites()) == 0, tile
if tile not in synth_tiles['tiles']:
synth_tiles['tiles'][tile] = {
'pins': [],
'loc': g.loc_of_tilename(tile),
}
synth_tiles['tiles'][tile]['pins'].append({
'roi_name': port['name'].replace('[', '_').replace(']','_'),
'wire': wire,
'pad': port['pin'],
'port_type': port_type,
'is_clock': is_clock,
})
with open(args.synth_tiles, 'w') as f:
json.dump(synth_tiles, f)
def dict_to_json_file(dicts, filename):
with open(filename, 'w') as filename:
json.dump(dicts, filename, sort_keys=True, indent=4)
def write_file(data):
# Writes recipes to the output file "output.txt" in json format
path = "output.txt"
f = open(path, "w")
json.dump(data, f, indent=2)
f.close()
def search(self):
# Set up the driver.
chrome_path = os.path.realpath('chromedriver')
chrome_options = Options()
chrome_options.add_experimental_option("detach", True)
driver = webdriver.Chrome(
executable_path=
'/Users/lyllayounes/Documents/lrn_github/ak_daily_scraping/chromedriver',
chrome_options=chrome_options)
page = driver.get('https://dps.alaska.gov/SORWeb/Registry/Search')
time.sleep(3)
# Query with nothing selected to view all entries.
button = driver.find_element_by_xpath(
'//*[@id="main_content"]/div/form/div/div[2]/div[7]/input[2]')
button.click()
# Wait for page load.
try:
driver.find_element_by_id('SearchResults_wrapper')
except:
time.sleep(1)
# Display all results
el = driver.find_element_by_xpath(
'//*[@id="SearchResults_length"]/label/select')
el.click()
for option in el.find_elements_by_tag_name('option'):
if (option.text == "All"):
option.click()
# Grab all the profile urls
profile_urls = []
links = driver.find_elements_by_xpath(
'//*[@id="SearchResults"]/tbody/tr[*]/td[*]/a')
for link in links:
profile_urls.append(link.get_attribute("href"))
# Set up data object to populate.
entries = {}
# FOR TESTING ONLY
# testurl = "https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=187137639149352769"
# profile_urls = [u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=793366663127834796', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=793356667327424794', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=646117133670656602', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=793376664127024792', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=793306663827784799', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=793356663927524795', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=793386663627104792', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=646158138570636606', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=793301667427314794', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=793336661727354798', u'https://dps.alaska.gov/SORWeb/registry/Detail?SexOffenderId=646177130270176607']
ctr = 0
for prof_url in profile_urls:
if (ctr % 20 == 0):
print("Processed " + str(ctr) + " of " +
str(len(profile_urls)) + " entries...")
entry = {}
try:
profile_html = requests.get(prof_url)
profile = BeautifulSoup(profile_html.text, "lxml")
row = profile.findAll("div", {"class": "row"})[1]
# Find name
name = row.find("h2").text
entry["name"] = name
# Find charge
charge = row.find("h3").text
entry["charge"] = charge
# Find the aliases
row = row.findAll("div")
aliases = []
for div in row:
aliases.append(div.text.replace('\n', ''))
entry["aliases"] = aliases
# Find the current status
status = profile.findAll("p")[0].text.strip(
) + " " + profile.findAll("p")[1].text.strip()
entry["status"] = status
# Get bottom panel information
panels = profile.findAll("div", {"class", "panel-default"})
# Find info for each panel
for i in range(0, 3):
personal_information = panels[
i] #.find("div", {"class", "panel-body"})
for row_div in personal_information.findAll(
"div", {"class", "row"}):
for div in row_div.findAll("div"):
if (len(div.text.split(":")) > 1):
entry[div.text.split(":")[0].strip(
)] = div.text.split(":")[1].strip()
except:
entry["profile_url"] = prof_url
entry["error"] = 1
entries[prof_url] = entry
# print(entries)
ctr += 1
with open("offender_information.txt", "w") as jsonfile:
json.dump(entries, jsonfile)
def parseData(itemList, outfile): # turns list to json
with open(outfile, 'w') as f:
json.dump(itemList, f)
def dump(*args, **kwargs):
if kwargs.pop('encode_datetime', False):
kwargs['default'] = _datetime_encoder
return json.dump(*args, **kwargs)
def write(self, version=VERSION):
fn = 'sankey_timeline.data.v4.js'.format(version)
with open(fn, 'w') as f:
json.dump(self.json, f, indent=2)
def zdkb(verbose=False,
listing=None,
fetch=False,
kb_dir=os.path.join(os.path.expanduser('~'), 'zdkb')):
"Manage Zendesk knowledgbases"
cfg = zdkb.getconfig()
# Log the cfg
if verbose:
print('Running with zdkb config:\n'
' verbose: {}\n'
' listing: {}\n'
' kb_dir: {}\n'.format(verbose, listing, kb_dir))
#if state['categories'] or state['topics'] or state['forums'] or state['list_zdf']:
if listing or fetch:
if cfg['zdesk_url'] and cfg['zdesk_email'] and cfg['zdesk_password']:
if verbose:
print('Configuring Zendesk with:\n'
' url: {}\n'
' email: {}\n'
' token: {}\n'
' password: (hidden)\n'.format(
cfg['zdesk_url'], cfg['zdesk_email'],
repr(cfg['zdesk_token'])))
zd = Zendesk(**cfg)
else:
msg = textwrap.dedent("""\
Config file (e.g. ~/.zdf2pdf.cfg) should be something like:
[zdf2pdf]
url = https://example.zendesk.com
mail = [email protected]
password = dneib393fwEF3ifbsEXAMPLEdhb93dw343
is_token = 1
""")
print(msg)
msg = textwrap.dedent("""\
Error: Need Zendesk config for requested operation.
Config file (~/.zdeskcfg) should be something like:
[zdesk]
url = https://example.zendesk.com
email = [email protected]
password = dneib393fwEF3ifbsEXAMPLEdhb93dw343
token = 1
[zdkb]
kb_dir = /path/to/kb_dir
""")
print(msg)
return 1
# If any listing was requested we will do that and exit, regardless of
# any other supplied options.
if listing == 'all':
# List available zendesk forums with their IDs and titles and exit.
# Listing is formatted like for following:
# 12345 Category 1 name
# 09876 Forum 1 name
# 54321 Forum 2 name
# 67890 Category 2 name
if verbose:
print('Listing all forums')
response = zd.forums_list(get_all_pages=True)
forums = response['forums']
cat_ids = set(f['category_id'] for f in forums)
categories = zd.categories_list(get_all_pages=True)
for cat_id in cat_ids:
if cat_id:
try:
cat_name = next(c['name'] for c in categories
if c['id'] == cat_id)
except StopIteration:
cat_name = 'None'
else:
cat_id = 'None'
cat_name = 'None'
print('{} ({})'.format(cat_name, cat_id))
for forum in forums:
if repr(forum['category_id']) == cat_id:
print(' {} ({})'.format(forum['name'], forum['id']))
return 0
elif listing:
if verbose:
print('Listing all entries in forum {}'.format(listing))
# List a zendesk forum's entries with their titles and IDs and exit
try:
forum_id = int(listing)
except ValueError:
print('Error: Could not convert to integer: {}'.format(listing))
return 1
entries = zd.forum_topics(id=listing, get_all_pages=True)
for entry in entries['topics']:
print('{} ({})'.format(entry['title'], entry['id']))
return 0
# Save the current directory so we can go back once done
start_dir = os.getcwd()
# Normalize all of the given paths to absolute paths
kb_dir = os.path.abspath(kb_dir)
# Check for and create working directory
if not os.path.isdir(kb_dir):
print('kb_dir does not exist: {}'.format(kb_dir))
return 1
if fetch:
# Change to working directory to begin file output
os.chdir(kb_dir)
response = zd.forums_list(get_all_pages=True)
forums = response['forums']
response = zd.categories_list(get_all_pages=True)
categories = response['categories']
response = zd.topics_list(get_all_pages=True)
topics = response['topics']
with open('categories', 'w') as cat_file:
json.dump(categories, cat_file)
for forum in forums:
forum_name = forum['name'].replace('/', '-')
if not os.path.isdir(forum_name):
os.mkdir(forum_name)
os.chdir(forum_name)
with open(forum_name + '.json', 'w') as forum_json:
json.dump(forum, forum_json)
os.chdir(kb_dir)
for topic in topics:
try:
forum_name = next(f['name'] for f in forums
if f['id'] == topic['forum_id'])
except StopIteration:
forum_name = 'None'
forum_name = forum_name.replace('/', '-')
if not os.path.isdir(forum_name):
os.mkdir(forum_name)
os.chdir(forum_name)
topic_filename = topic['title'].replace('/', '-')
with open(topic_filename + '.html', 'w') as topic_html:
topic_html.write(topic['body'])
with open(topic_filename + '.json', 'w') as topic_json:
del topic['body']
json.dump(topic, topic_json)
os.chdir(kb_dir)
return 0
return 0
def write_data(self):
with open(self.output_file, "w") as outfile:
json.dump(self.case_data, outfile)
config_file = './newtuber.json'
with open(config_file, 'r') as f:
ignore = simplejson.load(f)
print(os.environ.get('SENDGRID_API_KEY'))
print('here')
logging.info('Running')
for submission in r.get_subreddit('newtubers').get_new(limit=5):
if submission.link_flair_text == "GIVE CONTENT CRITIQUE":
if submission.id in ignore:
pass
else:
logging.info("New submission: %s" % submission.id)
ignore[submission.id] = submission.id
sg = sendgrid.SendGridAPIClient(
apikey=os.environ.get('SENDGRID_API_KEY'))
from_email = Email('*****@*****.**')
to_email = Email('*****@*****.**')
body = "New GIVE CONTENT CRITIQUE: <a href=" + submission.url + ">" + submission.title + "</a>"
content = Content('text/html', body)
subject = "New GIVE CONTENT CRITIQUE submission" + submission.title
mail = Mail(from_email, subject, to_email, content)
resp = sg.client.mail.send.post(request_body=mail.get())
print(resp.status_code)
print(resp.body)
print(resp.headers)
pass
with open(config_file, 'w') as wh:
simplejson.dump(ignore, wh)
for filename in tqdm(filenames):
with open(filename, 'r+') as filehandle:
try:
incident = simplejson.load(filehandle)
except:
logging.warning("Unable to load {0}.".format(filename))
continue
logging.debug("Before parsing:\n" + pformat(incident))
# fixsource
incident = subsource.fix_subsource(incident)
if cfg['fix']:
if overwrite:
filehandle.seek(0)
simplejson.dump(incident,
filehandle,
sort_keys=True,
indent=2,
separators=(',', ': '))
filehandle.truncate()
else:
with open(
cfg['output'].rstrip("/") + "/" +
filename.split("/")[-1], 'w') as outfilehandle:
simplejson.dump(incident,
outfilehandle,
sort_keys=True,
indent=2,
separators=(',', ': '))
logging.debug("After parsing:\n" + pformat(incident))
logging.info('Ending main loop.')
def test_dump(self):
sio = StringIO()
json.dump({}, sio)
self.assertEqual(sio.getvalue(), '{}')
if dat['extra'] == msg:
sim['extra']['LG'].append(dat)
for msg in ash['extra']['lg']:
for dat in data.values():
if dat['type'] == 'extra':
if dat['extra'] == msg:
ash['extra']['LG'].append(dat)
ash['ind']['wpm'] = float(ash['ind']['word']) / ash['ind']['msg']
ash['ind']['cpw'] = float(ash['ind']['char']) / ash['ind']['word']
ash['extra']['wpm'] = float(ash['extra']['word']) / ash['extra']['msg']
ash['extra']['cpw'] = float(ash['extra']['char']) / ash['extra']['word']
sim['ind']['wpm'] = float(sim['ind']['word']) / sim['ind']['msg']
sim['ind']['cpw'] = float(sim['ind']['char']) / sim['ind']['word']
sim['extra']['wpm'] = float(sim['extra']['word']) / sim['extra']['msg']
sim['extra']['cpw'] = float(sim['extra']['char']) / sim['extra']['word']
data = {'ash': ash, 'sim': sim}
# for x in [ash['ind'],ash['extra'],sim['ind'],sim['extra']]:
# for y in x:
# print(y,":",x[y])
#
json.dump(data, file, indent=4)
# msg=''
# for dat in data.values():
# if dat['type']=='msg':
# if dat['name'] == 'Ashish Jain' or dat['name'] in ' Ashish':
# if len(dat['msg'])>len(msg):
# msg=dat['msg']
#
# print(msg,len(msg))
database = {}
x = 0
with open("database.json", "r") as file:
database = simplejson.load(file)
for comment in subreddit.stream.comments(skip_existing=True):
comment_body = comment.body.split()
for word in comment_body:
for punction in string.punctuation:
if punction in word:
word.replace(punction, "") # Sanatize inputs
if not word in database:
database.setdefault(word, {})
database[word].setdefault("TOTAL_COUNT", 0)
if not comment.subreddit.display_name in database[word]:
database[word].setdefault(comment.subreddit.display_name, 0)
database[word]["TOTAL_COUNT"] += 1
database[word][comment.subreddit.display_name] += 1
if x == 1000:
print("Dumping to file ...")
with open("database.json", "w") as file:
simplejson.dump(database, file, indent=" ")
x = 0
print("Dumped!")
x += 1
def configure():
"""Super inefficient way to set up configuration, but I'm too lazy to do something better"""
import simplejson as json
input(
"Welcome to the settings menu for Eggbot. Press enter to start configuration. \n"
"If you did not intend to edit settings, please exit this process.")
with open('config.json', 'r') as cfg:
config = json.load(cfg)
print('Press enter to skip an option (leave it as is/go to default).')
one = True
while one:
logging = input(
'Do you want all messages visible to the bot to be printed in terminal? (y/n)\n'
).lower()
if logging in ('y', 'n', ''):
if logging == 'y':
config["logging"] = 1
elif logging == 'n':
config["logging"] = 0
else:
try:
config["logging"] = config["logging"]
except KeyError:
config["logging"] = 0
one = False
two = True
while two:
logging = input(
'Do you want all direct messages to the bot to be printed in terminal? (y/n)\n'
).lower()
if logging in ('y', 'n', ''):
if logging == 'y':
config["dmLog"] = 1
elif logging == 'n':
config["dmLog"] = 0
else:
try:
config["dmLog"] = config["dmLog"]
except KeyError:
config["dmLog"] = 1
two = False
three = True
while three:
logging = input(
'Do you want the terminal to log usage of Eggbot admin commands? (y/n)\n'
).lower()
if logging in ('y', 'n', ''):
if logging == 'y':
config["audit"] = 1
elif logging == 'n':
config["audit"] = 0
else:
try:
config["audit"] = config["audit"]
except KeyError:
config["audit"] = 1
three = False
four = True
while four:
logging = input(
'Do you want the terminal to log deleted messages? (y/n)\n').lower(
)
if logging in ('y', 'n', ''):
if logging == 'y':
config["deleteLog"] = 1
elif logging == 'n':
config["deleteLog"] = 0
else:
try:
config["deleteLog"] = config["deleteLog"]
except KeyError:
config["deleteLog"] = 1
four = False
with open('config.json', 'w') as cfg:
json.dump(config, cfg)
input("Settings have been saved. Press enter to proceed.")
return config
def commit_cache(self):
json.dump(self.cache, open(self.cache_file, 'w'))
def json_dump(dir, objs):
os.makedirs(dir, exist_ok=True)
for obj in objs:
with open('%s/%s.json' % (dir, obj['domain']), 'wt',
encoding='utf-8') as f:
json.dump(obj, f, indent=2, ensure_ascii=False)
def on_data(self, data):
with open('Tweets1.1.csv', 'a') as outfile:
json.dump(data, outfile)
return True
def get_material_details(url):
data = {}
cache_file_name = '%s.json' % hashlib.md5(url).hexdigest()
cache_file_path = os.path.join(cache_path, cache_file_name)
if xbmcvfs.exists(cache_file_path):
fp = open(cache_file_path, 'r')
data = json.load(fp)
fp.close()
return data
http = client.GET(url, httpSiteUrl)
if http is None:
return data
cover_regexp = re.compile("url\s*\(([^\)]+)")
beautifulSoup = BeautifulSoup(http)
info = beautifulSoup.find('div', 'item-info')
genre_element_container = info.findAll('span', {"itemprop": "genre"})
genres = []
for genre_element in genre_element_container:
genres.append(
strutils.fix_string(genre_element.find('span').string.strip()))
title = strutils.fix_string(
beautifulSoup.find('div',
'b-tab-item__title-inner').find('span').string)
original_title = strutils.html_entities_decode(
beautifulSoup.find('div', 'b-tab-item__title-origin').string)
description = beautifulSoup.find('p',
'item-decription').string.encode('utf-8')
poster = fs_ua.poster(
client.get_full_url(
beautifulSoup.find('div', 'poster-main').find('img')['src']))
images_container = beautifulSoup.find('div', 'b-tab-item__screens')
image_elements = images_container.findAll('a')
images = []
for image_element in image_elements:
images.append(
client.get_full_url(
fs_ua.poster(
cover_regexp.findall(str(
image_element['style']).strip())[0])))
rating_positive = beautifulSoup.find(
'div', 'm-tab-item__vote-value_type_yes').string.strip()
rating_negative = beautifulSoup.find(
'div', 'm-tab-item__vote-value_type_no').string.strip()
data = {
'title': title.strip(),
'original_title': original_title.strip(),
'poster': poster,
'description': description,
'images': images,
'genres': genres,
'rating_positive': rating_positive,
'rating_negative': rating_negative
}
fp = open(cache_file_path, 'w')
json.dump(data, fp)
fp.close()
return data
