def prepare_env(cnt, file_name=True, inline=False):
this = {}
if file_name:
this['file_name_json'] = os.path.join(SAMPLES_PATH, '{}.json'.format(cnt))
this['file_name_rst'] = os.path.join(SAMPLES_PATH,
'{}{inline}.rst'.format(cnt, inline='_inline' if inline else '')
)
with codecs.open(this['file_name_json'], 'r', encoding='utf-8') as _file:
doc = json.load(_file)
else:
this['file_name_json'] = False
this['file_name_rst'] = False
doc = json.load(cnt)
this['swagger_doc'] = rst.SwaggerObject(doc)
doc_module = importlib.import_module('swg2rst.utils.rst')
jinja_env = Environment(lstrip_blocks=True, trim_blocks=True)
jinja_env.loader = PackageLoader('swg2rst')
for name, function in inspect.getmembers(doc_module, inspect.isfunction):
jinja_env.filters[name] = function
jinja_env.filters['sorted'] = sorted
template = jinja_env.get_template('main.rst')
this['raw_rst'] = template.render(doc=this['swagger_doc'], inline=inline)
this['pattern'] = re.compile(r'[idm]_\w{32}')
this['normalize'] = lambda x: x[:-1] if x[-1] == '\n' else x
return this
def query_api(path, limit=None, **kwargs):
"""Returns an iterator over the objects returned from the govtrack.us API.
All additional keyword paramaters are passed onto the govtrack.us API in
the form of key/value pairs in the querystring of the URL. For further
information on the parameters available see the govtrack.us API docs at
http://www.govtrack.us/developers/api.
Arguments:
path -- the filepath of the API endpoint to query
Keyword Arguments:
limit -- the number of objects to return. If None (default), all objects
are returned
"""
url = GovTrackURL(path=path, **kwargs)
while True:
response = json.load(urllib2.urlopen(url.to_string()))
meta = response['meta']
objects = response['objects']
offset = meta['offset']
limit = limit if limit is not None else meta['total_count']
for obj in objects:
offset += 1
progress = int((float(offset)/limit) * 100)
obj_url = GovTrackURL(path='%s/%s' % (path, obj['id']))
yield json.load(urllib2.urlopen(obj_url.to_string())), progress
# If the limit argument is set, exit after we've reached that limit
if limit is not None and offset >= limit:
return
# Update the current offset value and, if we've retrieved all of the
# objects available, exit this function
url.offset = meta['offset'] + meta['limit']
def reverse_geocode(self, lon, lat):
params = {
'api_key': settings.API_KEYS['cloudmade'],
'lon': lon,
'lat': lat,
'type': 'road',
}
data = urllib2.urlopen(self.REVERSE_GEOCODE_URL % params)
json = simplejson.load(data)
if not json:
return []
else:
name = json['features'][0]['properties'].get('name')
try:
params['type'] = 'area'
data = simplejson.load(urllib2.urlopen(self.REVERSE_GEOCODE_URL % params))
name = '%s, %s' % (name, data['features'][0]['properties']['name'])
except Exception:
pass
return [{
'name': json['features'][0]['properties'].get('name'),
'location': (lon, lat),
'accuracy': 100,
}]
def POST(self, id=None, outputs=None, **kwargs):
if id is None:
metadata = self.parse_json_dict(simplejson.load(cherrypy.request.body))
for inp in metadata.get("inputs",[]):
if '_id' in inp: inp['_id'] = self.parse_value(inp['_id'])
id = Transformations.insert_one(metadata).inserted_id
# for inp in metadata.get("inputs",[]):
# Products.update_one({'_id':inp['_id']},{'$push':{'transformations':{'_id':id}}})
return self.build_response('ok', **dict(transformation={'_id':id}))
else:
if outputs == 'outputs':
metadata = self.parse_json_dict(simplejson.load(cherrypy.request.body))
resp = simplejson.loads(ProductsController().POST(id=None, metadata=metadata))
if resp['status'] != 'ok':
return self.build_response('error',
message='error creating output product: %s' % resp['message'])
Transformations.update_one({'_id':ObjectId(id)},{'$push':{'outputs':{'_id':ObjectId(resp['data']['product']['_id'])}}})
return self.build_response('ok', **dict(product=resp['data']['product']))
else:
return self.build_response('error',message='cannot POST metadata for an existing transformation. Use PUT instead.')
def generate(mapping, schema, filename):
mapping = json.load(open(mapping))
schema = json.load(open(schema))
schema.pop('$schema', 0)
# Index for which keys need to be marked as having exact counterparts
exact_index = get_exact_fields(mapping)
# Index to lookup a non-array version of key name (so it maps to ES)
es_items_index = generate_es_to_schema_index(schema)
# All the fields to process from the schema
schema_fields = get_schema_fields(schema)
for field in schema_fields:
type_field = field + '.' + 'type'
# No need expand these complex types with DEFAULT attributes
if get_deep(schema, type_field) in ['object', 'array']:
continue
if not field:
continue
es_field = es_items_index.get(field, None)
is_exact = True if es_field in exact_index else False
# Expand each of the keys to match DEFAULTS.
# set_deep() will not overwrite data if it does exist, but will add if it
# does not
for key, value in DEFAULTS.items():
full_key = field + '.' + key
if is_exact and key == 'is_exact':
value = True
set_deep(schema, full_key, value)
make_yaml(schema, filename)
def load():
states = simplejson.load(file(DATA_DIR + '/states/index.json'))
for code, state in states.iteritems():
if 'aka' in state: state.pop('aka')
db.insert('state', seqname=False, code=code, **unidecode(state))
districts = simplejson.load(file(DATA_DIR + '/districts/index.json'))
for name, district in districts.iteritems():
db.insert('district', seqname=False, name=name, **unidecode(district))
for fn in ['almanac', 'shapes', 'centers']:
print 'loading', fn
districts = simplejson.load(file(DATA_DIR + '/districts/%s.json' % fn))
for name, district in districts.iteritems():
db.update('district', where='name = $name', vars=locals(), **unidecode(district))
politicians = simplejson.load(file(DATA_DIR + '/politicians/index.json'))
for polid, pol in politicians.iteritems():
db.insert('politician', seqname=False, id=polid, **unidecode(pol))
for fn in ['govtrack', 'photos']:
print 'loading', fn
politicians = simplejson.load(file(DATA_DIR + '/politicians/%s.json' % fn))
for polid, pol in politicians.iteritems():
db.update('politician', where='id = $polid', vars=locals(), **unidecode(pol))
def index(request):
conn = urlopen( settings.RUNMODO_SOLR_URL + '/select?group=true&group.field=eventName&sort=raceStart%20desc&fq=raceStart:[*%20TO%20NOW]&rows=5')
results = simplejson.load(conn)
conn = urlopen( settings.RUNMODO_SOLR_URL + '/select?group=true&group.field=eventName&sort=raceStart%20asc&fq=raceStart:[NOW%20TO%20*]&rows=5')
events = simplejson.load(conn)
return render_to_response('runapp/index2.html', {'results': results['grouped']['eventName']['groups'], 'events': events['grouped']['eventName']['groups'], 'view':'index'}, context_instance=RequestContext(request) )
def _resume(results_dir):
"""Loads a partially completed json results directory."""
# Pylint can't infer that the json being loaded is a dict
# pylint: disable=maybe-no-member
assert os.path.isdir(results_dir), \
"TestrunResult.resume() requires a directory"
# Load the metadata
with open(os.path.join(results_dir, 'metadata.json'), 'r') as f:
meta = json.load(f)
assert meta['results_version'] == CURRENT_JSON_VERSION, \
"Old results version, resume impossible"
testrun = results.TestrunResult()
testrun.name = meta['name']
testrun.options = meta['options']
testrun.uname = meta.get('uname')
testrun.glxinfo = meta.get('glxinfo')
testrun.lspci = meta.get('lspci')
# Load all of the test names and added them to the test list
for file_ in os.listdir(os.path.join(results_dir, 'tests')):
with open(os.path.join(results_dir, 'tests', file_), 'r') as f:
try:
testrun.tests.update(json.load(f, object_hook=piglit_decoder))
except ValueError:
continue
return testrun
def _resume(results_dir):
"""Loads a partially completed json results directory."""
# Pylint can't infer that the json being loaded is a dict
# pylint: disable=maybe-no-member
assert os.path.isdir(results_dir), \
"TestrunResult.resume() requires a directory"
# Load the metadata
with open(os.path.join(results_dir, 'metadata.json'), 'r') as f:
meta = json.load(f)
assert meta['results_version'] == CURRENT_JSON_VERSION, \
"Old results version, resume impossible"
meta['tests'] = collections.OrderedDict()
# Load all of the test names and added them to the test list
tests_dir = os.path.join(results_dir, 'tests')
file_list = sorted(
(l for l in os.listdir(tests_dir) if l.endswith('.json')),
key=lambda p: int(os.path.splitext(p)[0]))
for file_ in file_list:
with open(os.path.join(tests_dir, file_), 'r') as f:
try:
meta['tests'].update(json.load(f))
except ValueError:
continue
return results.TestrunResult.from_dict(meta)
def suckdry_planlos():
import urllib
f = urllib.urlopen("http://planlosbremen.de/termine/service/monat")
#https://planlosbremen.de/termine/service/location/5
#jsondata = f.read()
data = simplejson.load(f)
u = db.User.find_one()
for i in data:
loc_id = i['fields']['location']
url = urllib.urlopen("http://planlosbremen.de/termine/service/location/%s" % loc_id)
loc = simplejson.load(url)
location = loc_name = loc[0]['fields']['name']
e = db.Event()
f = i['fields']
e.title = unicode(f['title'])
e.author = u
form_date = date_parser(f['datum']).date()
form_time = date_parser(f['time']).time()
e.eventdate = datetime.combine(form_date, form_time)
e.short_desc = unicode(f['short_desc'])
e.desc = unicode(f['desc'])
e.url = unicode(f['exturl'])
e.tags = [unicode(f['type'])]
e.is_published = f['is_pub']
e.save()
def run_compare():
with open('url-content-read.json') as f:
readability_text = simplejson.load(f)
with open('url-content-fetcher.json') as f:
fetcher_text = simplejson.load(f)
cnt = 0
z_cnt = 0
dmp = diff_match_patch()
rets = []
for key, value in readability_text.items():
if key in fetcher_text:
cnt += 1
rc = re.sub(r' ', '', value)
fc = re.sub(r' ', '', fetcher_text[key])
l_len = len(rc)
r_len = len(fc)
retval = dif_content(rc, fc)
retval_ground = 0
results = dmp.diff_main(rc, fc)
for res in results:
if res[0] == 0:
retval_ground += len(res[1])
print cnt, ': ', l_len, r_len, retval, retval_ground
real_ret = max(retval, retval_ground)
rets.append((cnt, l_len, r_len, real_ret))
with open('diff_result_1', 'w') as f:
for res in rets:
print >> f, res[0], ': ', res[1], res[2], res[3]
def test_jsonsink():
import simplejson
import os
ds = nio.JSONFileSink()
yield assert_equal, ds.inputs._outputs, {}
ds = nio.JSONFileSink(in_dict={'foo': 'var'})
yield assert_equal, ds.inputs.in_dict, {'foo': 'var'}
ds = nio.JSONFileSink(infields=['test'])
yield assert_true, 'test' in ds.inputs.copyable_trait_names()
curdir = os.getcwd()
outdir = mkdtemp()
os.chdir(outdir)
js = nio.JSONFileSink(infields=['test'], in_dict={'foo': 'var'})
js.inputs.new_entry = 'someValue'
setattr(js.inputs, 'contrasts.alt', 'someNestedValue')
res = js.run()
with open(res.outputs.out_file, 'r') as f:
data = simplejson.load(f)
yield assert_true, data == {"contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"}
js = nio.JSONFileSink(infields=['test'], in_dict={'foo': 'var'})
js.inputs.new_entry = 'someValue'
js.inputs.test = 'testInfields'
setattr(js.inputs, 'contrasts.alt', 'someNestedValue')
res = js.run()
with open(res.outputs.out_file, 'r') as f:
data = simplejson.load(f)
yield assert_true, data == {"test": "testInfields", "contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"}
os.chdir(curdir)
shutil.rmtree(outdir)
def query(self, object_id=None,type=None,connection_type=None, metadata=False):
response = {}
import urllib
import simplejson
if type == 'fb':
url = 'https://graph.facebook.com/%s' % (object_id)
if connection_type:
url += '/%s' % (connection_type)
params = {'access_token': self.access_token}
if metadata:
params['metadata'] = 1
url += '?' + urllib.urlencode(params)
response = simplejson.load(urllib.urlopen(url))
if 'error' in response:
error = response['error']
raise FacebookSessionError(error['type'], error['message'])
if type == 'github':
url = 'https://api.github.com/user'
params = {'access_token': self.access_token}
url += '?' + urllib.urlencode(params)
print "DAY LA URL GITHUB %s " % url
response = simplejson.load(urllib.urlopen(url))
return response
def update(self):
request = self.request
id = request.model_id
fs = self.get_fieldset(suffix="Edit", id=id)
event = events.BeforeRenderEvent(self.request.model_instance, self.request, fs=fs)
alsoProvides(event, events.IBeforeEditRenderEvent)
zope.component.event.objectEventNotify(event)
if request.format == "json" and request.method == "PUT":
data = json.load(request.body_file)
elif request.content_type == "application/json":
data = json.load(request.body_file)
else:
data = request.POST
with_prefix = True
if request.format == "json":
with_prefix = bool(request.params.get("with_prefix"))
fs = fs.bind(request=request, with_prefix=with_prefix)
if self.validate(fs):
fs.sync()
self.sync(fs, id)
self.session.flush()
if request.format in ("html", "xhr"):
if request.is_xhr or request.format == "xhr":
return Response(content_type="text/plain")
return exc.HTTPFound(location=request.fa_url(request.model_name, _pk(fs.model)))
else:
return self.render(fs=fs, status=0)
if request.format == "html":
return self.render(fs=fs, id=id)
else:
return self.render(fs=fs, status=1)
def read_config(self):
try:
logger.info("going to read the sites config file ")
with open(NewsCrawlerConfig.SITE_FILE, "rbU") as config:
try:
logger.info("going to json load the sites config")
json.load(config, object_hook = self.as_newscrawler)
self.is_config_read = True
except Exception as load_e:
logger.error(load_e)
raise load_e
except IOError as file_e:
logger.error(file_e)
raise file_e
try:
logger.info("going to read the crawler configurations")
with open(NewsCrawlerConfig.CRAWL_FILE, "rbU") as config:
try:
logger.info("going to json load the crawler configs")
json.load(config, object_hook = self.as_crawloptions)
except Exception as load_e:
logger.error(load_e)
raise load_e
except IOError as file_e:
logger.error(file_e)
raise file_e
def searchImg(query):
## reference: https://developers.google.com/image-search/v1/jsondevguide#json_snippets_python
query = string.replace(query," ","%20")
url = ("https://ajax.googleapis.com/ajax/services/search/images?" + "v=1.0&q=" + query + "&rsz=8")
request = urllib2.Request(url, None, {'Referer':"www.github.com/gradiuscypher/Neo"})
response = urllib2.urlopen(request)
results = simplejson.load(response)
## list of result pages, used for &start query
pagelist = results['responseData']['cursor']['pages']
pages = []
for i in pagelist:
pages.append(i['start'])
##return all results, save them to a dictionary, append that dictionary to a list
resultList = []
for i in pages:
query = string.replace(query," ","%20")
url = ("https://ajax.googleapis.com/ajax/services/search/images?" + "v=1.0&q=" + query + "&rsz=8&start=" + i)
request = urllib2.Request(url, None, {'Referer':"www.github.com/gradiuscypher/Neo"})
response = urllib2.urlopen(request)
results = simplejson.load(response)
anslist = results['responseData']['results']
for i in anslist:
resultList.append({'name':i['visibleUrl'],'url':i['url'],'titleNoFormatting':i['titleNoFormatting']})
return resultList
def getPages():
"""
For each page of the diary: get the json and save in ./Diaries/diaryID/pageID.json
"""
path = "Diaries/"
urlBase = "http://transcripts.sl.nsw.gov.au/api/node/"
errorsList = []
for diaryUrl in diariesUrls:
print diaryUrl
response = urllib2.urlopen(diaryUrl)
# print response.info()
myjson = simplejson.load(response)
pagesList = myjson["field_transcript_pages"]["und"]
## Create folder if it doesn't exist
directory = path + diaryUrl[42:-5]
if not os.path.exists(directory):
os.makedirs(directory)
## Get each page in json
for page in pagesList:
print page["nid"]
mypage = urlBase + page["nid"] + ".json"
try:
response = urllib2.urlopen(mypage)
# print response.info()
myjson = simplejson.load(response)
file = open(directory + "/" + page["nid"] + ".json", "w+")
file.write(str(myjson))
file.close()
except urllib2.HTTPError, e:
print "We failed with error code - %s." % e.code
print mypage
print "$$$$$$$$$$$$"
errorsList.append("diary:" + diaryUrl[42:-5] + ": page " + mypage)
def test_json_is_valid(self):
""" JSONBackend.write_test() produces valid json """
with open(os.path.join(self.tdir, 'tests', '0.json'), 'r') as f:
try:
json.load(f)
except Exception as e:
raise AssertionError(e)
def test_results_valid(self):
""" JSONBackend.finalize() results.json is valid """
with open(os.path.join(self.tdir, 'results.json'), 'r') as f:
try:
json.load(f)
except Exception as e:
raise AssertionError(e)
def _resume(results_dir):
"""Loads a partially completed json results directory."""
# TODO: could probably use TestrunResult.from_dict here
# Pylint can't infer that the json being loaded is a dict
# pylint: disable=maybe-no-member
assert os.path.isdir(results_dir), \
"TestrunResult.resume() requires a directory"
# Load the metadata
with open(os.path.join(results_dir, 'metadata.json'), 'r') as f:
meta = json.load(f)
assert meta['results_version'] == CURRENT_JSON_VERSION, \
"Old results version, resume impossible"
meta['tests'] = {}
# Load all of the test names and added them to the test list
for file_ in os.listdir(os.path.join(results_dir, 'tests')):
with open(os.path.join(results_dir, 'tests', file_), 'r') as f:
try:
meta['tests'].update(json.load(f, object_hook=piglit_decoder))
except ValueError:
continue
return results.TestrunResult.from_dict(meta)
def test_payload_splitter(self):
with open(FIXTURE_PATH + '/legacy_payload.json') as f:
legacy_payload = json.load(f)
legacy_payload_split, metrics_payload, checkruns_payload = split_payload(dict(legacy_payload))
series = metrics_payload['series']
legacy_payload_split['metrics'] = []
for s in series:
attributes = {}
if s.get('type'):
attributes['type'] = s['type']
if s.get('host'):
attributes['hostname'] = s['host']
if s.get('tags'):
attributes['tags'] = s['tags']
if s.get('device'):
attributes['device_name'] = s['device']
formatted_sample = [s['metric'], s['points'][0][0], s['points'][0][1], attributes]
legacy_payload_split['metrics'].append(formatted_sample)
del legacy_payload['service_checks']
self.assertEqual(legacy_payload, legacy_payload_split)
with open(FIXTURE_PATH + '/sc_payload.json') as f:
expected_sc_payload = json.load(f)
self.assertEqual(checkruns_payload, expected_sc_payload)
def load_data(dump_file):
client = pymongo.MongoClient("localhost", 27017)
db = client['courses']
coursesd = db['coursesd']
sectionsd = db['sectionsd']
doc_queue = []
with open(dump_file) as f:
for course in json.load(f):
pairs = [(name, _typify(course.get(name), data_type)) for (name,
data_type) in course_schema if name not in special_fields]
pairs += _special_treatment(course, course_schema)
doc = dict(pairs)
coursesd.update({'CourseFull': doc['CourseFull']},
doc, True)
print '%d courses in db.' % coursesd.count()
with open(dump_file) as f:
for course in json.load(f):
pairs = [(name, _typify(course[name], data_type)) for (name,
data_type) in section_schema if name not in special_fields]
pairs += _special_treatment(course, section_schema)
doc = dict(pairs)
doc_queue.append(doc)
if len(doc_queue) == 100:
inserted = sectionsd.insert(doc_queue)
print '%d documents inserted' % len(inserted)
doc_queue = []
if doc_queue:
inserted = sectionsd.insert(doc_queue)
print '%d documents inserted' % len(inserted)
doc_queue = []
print '%d sections in db.' % sectionsd.count()
def __init__(self, authtype = "OAuth", username = None, password = None, oauth_keys = None, headers = None):
self.authtype = authtype
self.authenticated = False
self.username = username
self.password = password
self.oauth_keys = oauth_keys
if self.username is not None and self.password is not None:
if self.authtype == "OAuth":
self.request_token_url = 'https://twitter.com/oauth/request_token'
self.access_token_url = 'https://twitter.com/oauth/access_token'
self.authorization_url = 'http://twitter.com/oauth/authorize'
self.signin_url = 'http://twitter.com/oauth/authenticate'
# Do OAuth type stuff here - how should this be handled? Seems like a framework question...
elif self.authtype == "Basic":
self.auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
self.auth_manager.add_password(None, "http://twitter.com", self.username, self.password)
self.handler = urllib2.HTTPBasicAuthHandler(self.auth_manager)
self.opener = urllib2.build_opener(self.handler)
if headers is not None:
self.opener.addheaders = [('User-agent', headers)]
try:
simplejson.load(self.opener.open("http://twitter.com/account/verify_credentials.json"))
self.authenticated = True
except HTTPError, e:
raise TangoError("Authentication failed with your provided credentials. Try again? (%s failure)" % `e.code`, e.code)
def BestMatch(sname, api_code):
url = "https://opencorporates.com/reconcile/?query="+urllib.quote(sname)
entities = simplejson.load(urllib.urlopen(url))
try:
bm = entities['result'][0]
url = "https://api.opencorporates.com" + bm['id'] + '?api_token='+ api_code
match = simplejson.load(urllib.urlopen(url))
m = match['results']['company']
if m['controlling_entity'] == None:
cen = ''
cec = ''
ceu = ''
else:
cen = m['controlling_entity']['name']
cec = m['controlling_entity']['jurisdiction_code']
ceu = m['controlling_entity']['opencorporates_url']
print sname, m['name'], m['jurisdiction_code'], m['company_number'], m['corporate_groupings'], m['agent_name'], m['agent_address'], m['alternative_names'], m['previous_names'], m['home_company'], cen, cec, ceu, m['inactive'], bm['score'], bm['match'], bm['uri'], m['registry_url']
reconciled.writerow([sname, m['name'].encode('utf-8'), m['jurisdiction_code'], m['company_number'], m['corporate_groupings'], m['agent_name'], m['agent_address'], m['alternative_names'], m['previous_names'], m['home_company'], cen, cec, ceu, m['inactive'], bm['score'], bm['match'], bm['uri'], m['registry_url']])
return match['results']['company']['name']
except IndexError:
reconciled.writerow([sname, 'nomatch'])
return "nomatch"
def tz(phenny, input):
"""Displays time in different time zones"""
with open('nickloc.csv', 'rU') as f:
z = csv.reader(f)
nickdict = {}
for key, val in z:
nickdict[key] = val
try:
locinput = input.group(2)
locinput1 = locinput.strip().lower()
if locinput1 in nickdict:
htmlinput = urllib.quote(nickdict[locinput1])
else:
htmlinput = urllib.quote(locinput1)
url2 = 'http://nominatim.openstreetmap.org/search?q=' + htmlinput + '&format=json'
jsonResponse = simplejson.load(urllib.urlopen(url2))
lati = jsonResponse[0]['lat']
longi = jsonResponse[0]['lon']
loca = jsonResponse[0]['display_name']
url3 = 'http://api.geonames.org/timezoneJSON?lat=' + lati + '&lng=' + longi + '&username=jinpark'
jsonResponse1 = simplejson.load(urllib.urlopen(url3))
if jsonResponse1['dstOffset'] == 0:
timezone = ''
elif jsonResponse1['dstOffset'] > 0:
timezone = '+' + str(jsonResponse1['dstOffset'])
else:
timezone = str(jsonResponse1['dstOffset'])
phennyout = loca + ": " + str(jsonResponse1['time']) + ' UTC' + timezone
phenny.say(phennyout)
except:
phenny.say('Something went wrong')
def apply_array_location_correction(phot_groups_filepath):
"""
Reads the phot_groups.json files and applies the array
location correction to all measurements of ALL sources.
Writes the result to disk in work_dir as
'phot_groups_arrayloc.json'
"""
work_dir = "/".join(phot_groups_filepath.split("/")[:-1])
meta = json.load(open(work_dir + "/metadata.json"))
# read in the array location correction values
root = os.path.abspath("../bcdphot")
if meta["channel"] is "1":
fp = os.path.join(root, "cal", "ch1_photcorr_ap_5.fits")
elif meta["channel"] is "2":
fp = os.path.join(root, "cal", "ch2_photcorr_ap_5.fits")
arrloc = pyfits.open(fp)[0].data
# read in the photometry JSON files
ch = json.load(open(phot_groups_filepath))
# apply correction
for key in ch:
for obs in ch[key]:
x, y = obs[4:6]
obs[6:] = [i * arrloc[x, y] for i in obs[6:]]
# write to disk
out_path = work_dir + "/phot_groups_arrayloc.json"
with open(out_path, "w") as w:
json.dump(ch, w, indent=4 * " ")
print("created file: " + out_path)
def search(query, resultnum=None, lang=None, since_id=None):
output = []
q = []
rpp = 100
q.append(urlencode({'q': query}))
if not since_id is None:
q.append(urlencode({'since_id': since_id}))
if not lang is None:
q.append(urlencode({'lang': lang}))
if not resultnum is None:
rpp = resultnum
q.append(urlencode({'rpp': rpp}))
baseurl = 'http://search.twitter.com/search.json'
url = baseurl + '?' + '&'.join(q)
print url
response = urllib2.urlopen(url)
data = simplejson.load(response)
output.append(data)
while 'next_page' in data:
url = baseurl + data['next_page']
print url
response = urllib2.urlopen(url)
data = simplejson.load(response)
output.append(data)
return output
def __init__(self, resultfile=None):
self.serialized_keys = ['options',
'name',
'tests',
'uname',
'wglinfo',
'glxinfo',
'lspci',
'time_elapsed']
self.name = None
self.uname = None
self.options = None
self.glxinfo = None
self.lspci = None
self.time_elapsed = None
self.tests = {}
if resultfile:
# Attempt to open the json file normally, if it fails then attempt
# to repair it.
try:
raw_dict = json.load(resultfile)
except ValueError:
raw_dict = json.load(self.__repairFile(resultfile))
# Check that only expected keys were unserialized.
for key in raw_dict:
if key not in self.serialized_keys:
raise Exception('unexpected key in results file: ', str(key))
self.__dict__.update(raw_dict)
# Replace each raw dict in self.tests with a TestResult.
for (path, result) in self.tests.items():
self.tests[path] = TestResult(result)
def get_manifest(tarfile_path):
"""Extract and return manifest in tarball.
Given a path to a tarfile, which is itself the product of "docker save",
this discovers the manifest for the collection of images, which provides
version information.
"""
manifest = []
configjson = None
with contextlib.closing(tarfile.open(tarfile_path)) as archive:
for member in archive.getmembers():
# find the "manifest.json" file, which points to metadata json file
if os.path.basename(member.path) == 'manifest.json':
initial_manifest = json.load(archive.extractfile(member))
configjson = initial_manifest[0]['Config']
for member in archive.getmembers():
# get manifest from shasum json file, docker ver > 1.10
if configjson and os.path.basename(member.path) == configjson:
image_data = json.load(archive.extractfile(member))
manifest.append(image_data)
# find the "json" files, which contain all image metadata
# legacy code for docker ver < 1.10
elif not configjson and os.path.basename(member.path) == 'json':
image_data = json.load(archive.extractfile(member))
manifest.append(image_data)
return manifest
def showkeys(d,root=''):
import simplejson
import urllib
if d == '-':
d = simplejson.load(sys.stdin)
elif isinstance(d,str):
d = simplejson.load(urllib.urlopen(d))
for k,v in d.items():
if isinstance(v,str):
print "%s[%s]=%r" % (root,k,v)
if isinstance(v,int):
print "%s[%s]=%r" % (root,k,v)
if isinstance(v,float):
print "%s[%s]=%r" % (root,k,v)
if isinstance(v,dict):
showkeys(v,root = root + '['+k+']')
if isinstance(v,list):
for i,v in enumerate(v):
showkeys(v,root = root + '['+k+']' + '['+str(i)+']')
from urllib.request import urlopen
import simplejson as json
ChannelID = 'UCCJdqnlqrmTUAYvS6i3nAXA'
DeveloperKey = "PUT YOUR KEY HERE"
maxResults = '50'
CommentsList = []
CommentsUrl = "https://www.googleapis.com/youtube/v3/commentThreads?key=" + DeveloperKey + "&channelId=" + ChannelID + "&part=snippet&maxResults=" +maxResults + "&textFormat=plainText"
Results = urlopen(CommentsUrl)
Results = json.load(Results)
for searchResult in Results.get("items", []):
if 'topLevelComment' in searchResult['snippet']:
comment = searchResult['snippet']["topLevelComment"]
author = comment["snippet"]["authorDisplayName"]
text = comment["snippet"]["textDisplay"]
print (author, text)
CommentsList.append(text)
print(CommentsList)
#!/usr/bin/env python
# Copyright (c) 2015 SpinPunch. All rights reserved.
# Use of this source code is governed by an MIT-style license that can be
# found in the LICENSE file.
import simplejson as json
gamedata = json.load(open('gamedata.json'))
freetime = gamedata['store']['free_speedup_time']
def alter_time(t):
newt = int(t / 2)
# do not create times that are close to the free production time limit
# (which often causes order confusion), instead push it under
if newt > (freetime - 15) and newt < (freetime + 15):
newt = (freetime - 15)
return newt
if __name__ == '__main__':
ret = {}
for name, unit in gamedata['units'].iteritems():
old_times = unit['build_time']
if type(old_times) == list:
new_times = map(alter_time, old_times)
else:
new_times = alter_time(old_times)
# data_paths = ['.']
# pass
# if "input" in cfg:
# cfg["input"] = [l.strip() for l in cfg["input"].split(" ,")] # spit to list
# else:
# raise ValueError("No input director or file provided to validate.")
# files_to_validate = set()
incident_counter = 0
for src in cfg["input"]:
if os.path.isfile(src):
logging.debug("Now validating {0}.".format(src))
# errors in json
try:
incident = simplejson.load(open(src))
except simplejson.scanner.JSONDecodeError:
logging.warning("ERROR: %s did not parse properly. Skipping" %
src)
# replacing vakudate() with iterating errors - 171206 - GDB
for e in validator.iter_errors(incident):
offendingPath = '.'.join(str(x) for x in e.path)
logging.warning("ERROR in %s. %s %s" %
(src, offendingPath, e.message))
# capture errors from main
# try:
# # validator.validate(incident) # replacing with iterating errors - 171206 GDB
# main(incident)
# except ValidationError as e:
# offendingPath = '.'.join(str(x) for x in e.path)
# logging.warning("ERROR in %s. %s %s" % (src, offendingPath, e.message))
import os
import sys
from bottle import run, Bottle, static_file, request
from server.params import params, jsonabort
from server.staticroute import staticroutestack
import toolz as t
import simplejson as json
db = json.load(open('data/db.json'))
# all the fields present in a production
skeleton = {
# constant fields (don't get updated)
'title': '',
'author': '',
'characters': {},
'script': [],
# modifiable fields - can change these fields.
'blocking': {},
'cues': {},
'line_notes': {},
'director_notes': {},
}
app = Bottle()
@app.get('/')
def index():
return static_file('index.html', 'client')
def sense_tasks(request, senseId):
user = request.user
if not user or user not in ADMINS:
raise Forbidden('not an admin')
sense = get_database().senses.find_one({'_id': senseId})
(old_tasks, new_tasks) = query_tasks(request.corpus, sense['lemma'],
sense['_id'], sense.get('pos', 'N'))
if request.method == 'GET':
# retrieve task statistics
info = {
'num_existing': len(old_tasks),
'num_remaining': len(new_tasks)
}
return Response(json.dumps(info), mimetype="text/javascript")
elif request.method == 'POST':
stuff = json.load(request.stream)
print >> sys.stderr, stuff
if stuff['method'] == 'existing':
for task in old_tasks:
task.set_annotators(
sorted(
set(task._doc.get('annotators',
[])).union(stuff['annotators'])))
info = {
'num_existing': len(old_tasks),
'num_remaining': len(new_tasks)
}
return Response(json.dumps(info), mimetype="text/javascript")
elif stuff['method'] == 'remaining':
for task in new_tasks:
create_new_task(task)
task.set_annotators(
sorted(
set(task._doc.get('annotators',
[])).union(stuff['annotators'])))
task.save()
info = {
'num_existing': len(old_tasks) + len(new_tasks),
'num_remaining': 0
}
return Response(json.dumps(info), mimetype="text/javascript")
elif stuff['method'] == 'wsdgold':
spans = set()
task0 = old_tasks[0]
for task in old_tasks:
if 'wsdgold' not in task._doc.get('annotators', []):
spans.update(adjudication_spans(task))
task.set_annotators(
sorted(
set(task._doc.get('annotators',
[])).union(['wsdgold'])))
task.set_status('wsdgold', 'ready')
task.save()
print >> sys.stderr, spans
if spans:
task_adj = create_adjudication_task(task0, spans)
task_adj.save()
info = {
'num_existing': len(old_tasks),
'num_remaining': len(spans)
}
return Response(json.dumps(info), mimetype="text/javascript")
print >> sys.stderr, 'huh?'
def __init__(self,
serverPushCb,
queue=None,
path=None,
filter=True,
bufferDelay=1,
retryDelay=5,
blackList=None):
"""
@serverPushCb: callback to be used. It receives 'self' as parameter. It
should call self.queueNextServerPush() when it's done to queue the next
push. It is guaranteed that the queue is not empty when this function is
called.
@queue: a item queue that implements IQueue.
@path: path to save config.
@filter: when True (default), removes all "", None, False, [] or {}
entries.
@bufferDelay: amount of time events are queued before sending, to
reduce the number of push requests rate. This is the delay between the
end of a request to initializing a new one.
@retryDelay: amount of time between retries when no items were pushed on
last serverPushCb call.
@blackList: events that shouldn't be sent.
"""
StatusReceiverMultiService.__init__(self)
# Parameters.
self.watched = []
self.queue = queue
if self.queue is None:
self.queue = MemoryQueue()
self.queue = IndexedQueue(self.queue)
self.path = path
self.filter = filter
self.bufferDelay = bufferDelay
self.retryDelay = retryDelay
if not callable(serverPushCb):
raise NotImplementedError('Please pass serverPushCb parameter.')
def hookPushCb():
# Update the index so we know if the next push succeed or not, don't
# update the value when the queue is empty.
if not self.queue.nbItems():
return defer.succeed(None)
self.lastIndex = self.queue.getIndex()
return defer.maybeDeferred(serverPushCb, self)
self.serverPushCb = hookPushCb
self.blackList = blackList
# Other defaults.
# IDelayedCall object that represents the next queued push.
self.task = None
self.stopped = False
self.lastIndex = -1
self.state = {
'started': str(datetime.datetime.utcnow()),
'next_id': 1,
'last_id_pushed': 0
}
# Try to load back the state.
if self.path and os.path.isdir(self.path):
state_path = os.path.join(self.path, 'state')
if os.path.isfile(state_path):
with open(state_path, 'r') as f:
self.state.update(json.load(f))
if self.queue.nbItems():
# Last shutdown was not clean, don't wait to send events.
self.queueNextServerPush()
def taskcluster_upload(self):
auth = os.path.join(os.getcwd(),
self.config['taskcluster_credentials_file'])
credentials = {}
execfile(auth, credentials)
client_id = credentials.get('taskcluster_clientId')
access_token = credentials.get('taskcluster_accessToken')
if not client_id or not access_token:
self.warning(
'Skipping S3 file upload: No taskcluster credentials.')
return
self.activate_virtualenv()
dirs = self.query_abs_dirs()
locales = self.query_locales()
make = self.query_exe("make")
upload_env = self.query_upload_env()
cwd = dirs['abs_locales_dir']
branch = self.config['branch']
revision = self.query_revision()
repo = self.query_l10n_repo()
pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool')
pushdate = time.strftime('%Y%m%d%H%M%S',
time.gmtime(pushinfo.pushdate))
routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'],
'testing/taskcluster/routes.json')
with open(routes_json) as routes_file:
contents = json.load(routes_file)
templates = contents['l10n']
for locale in locales:
output = self.get_output_from_command_m(
"%s echo-variable-UPLOAD_FILES AB_CD=%s" % (make, locale),
cwd=cwd,
env=upload_env,
)
files = shlex.split(output)
abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files]
routes = []
fmt = {
'index':
self.config.get('taskcluster_index', 'index.garbage.staging'),
'project':
branch,
'head_rev':
revision,
'pushdate':
pushdate,
'year':
pushdate[0:4],
'month':
pushdate[4:6],
'day':
pushdate[6:8],
'build_product':
self.config['stage_product'],
'build_name':
self.query_build_name(),
'build_type':
self.query_build_type(),
'locale':
locale,
}
for template in templates:
routes.append(template.format(**fmt))
self.info('Using routes: %s' % routes)
tc = Taskcluster(
branch,
pushinfo.pushdate, # Use pushdate as the rank
client_id,
access_token,
self.log_obj,
)
task = tc.create_task(routes)
tc.claim_task(task)
for upload_file in abs_files:
tc.create_artifact(task, upload_file)
tc.report_completed(task)
#!/usr/bin/env python3 #-*- coding:utf-8 -*- #********************************************************************************************************* #* __ __ __ ______ __ __ _______ _______ * #* | |--.| |.---.-..----.| |--.| __ \.---.-..-----.| |_| |--..-----..----. | | __| * #* | _ || || _ || __|| < | __/| _ || || _| || -__|| _| | - |__ | * #* |_____||__||___._||____||__|__||___| |___._||__|__||____|__|__||_____||__| |_______|_______| * #* http://www.blackpantheros.eu | http://www.blackpanther.hu - kbarcza[]blackpanther.hu * Charles Barcza * #*************************************************************************************(c)2002-2018******** # This is very useful tool if you lost exactly passwd for your Encfs. # Written by Miklos Horvath - [email protected] # the demo passwd is "#EncFS$Rec@ver!123" import itertools import simplejson with open("COMBS", "r") as fp: combs = itertools.product(*simplejson.load(fp)) for e in combs: pwd = "".join(e) print(pwd) # use for test and stop if generated your demo key, just add XXX to key to ignore this line, or delete XXX for use for test if pwd == "XXX#EncFS$Rec@ver!123": print("-- OK -- Last key is your generated demo key!") break
#判斷該路徑的檔案是否存在或是否有資料
if os.path.isfile(file_path) and os.stat(file_path).st_size != 0:
#將json檔的資料寫入txt檔內
#雖然他多寫r+，但實際結果感覺一樣
#old_file = open(file_path,"r+")
old_file = open(file_path)
#json.loads()：從json檔讀資料後，將其從str轉換成dictionary
#順帶一提，read會回傳str
#data = json.loads(old_file.read())
#json.load()：直接從json檔讀資料，因此，可以說他具備read()的功能
data = json.load(old_file)
write_into(data)
else:
#如果json檔不存在，就新增一個，然後幫他寫入資料
old_file = open(file_path,"w+")
#資料要寫成這樣，才能存入，但有時還是會出現simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
data = {"ages":20,
"sex":"male"}
#以json檔的資料格式寫入新創造的json檔
#json.dumps則是單純將資料轉乘json格式並回傳資料
old_file.write(json.dumps(data))
import numpy as np
VOCAB_SIZE = 2000
TRAINING_FILES = 100
BATCH_SIZE = 256
EPOCHS = 2000
DIMENSIONS = 50
#after how many training samples to save model
SAVE_MODEL = 10000
samples = 0
count = 0
w2v = train_word_vectors(VOCAB_SIZE, BATCH_SIZE, EPOCHS, DIMENSIONS)
for i in range(TRAINING_FILES):
f = open('Samples/samples_{}'.format(i), 'r')
data = simplejson.load(f)
x_train = np.asarray(data[0])
y_train = np.asarray(data[1])
#print x_train.shape
#print y_train.shape
del data[:]
count += len(x_train)
if(count >= SAVE_MODEL):
print 'Saved model after {} samples'.format(count)
w2v.save('w2v.h5')
count = 0
cbk = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=0, mode='min')
w2v.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.2, callbacks=[cbk], verbose=1)
samples += len(x_train)
print "Trained on {} samples".format(samples)
from urllib2 import *
from solr import *
import urllib2
import logging
import simplejson
from stemming.porter2 import stem
logging.basicConfig(format=' %(message)s', level=logging.INFO)
from gensim import corpora, models, similarities
#conn = urlopen('http://richieverma.koding.io:8983/solr/partB/select?q=tweet_tags:*&wt=json&rows=13000')
conn = urlopen('http://52.34.17.82:8983/solr/ezra/select?q=tweet_tags:*&wt=json&rows=11051')
initial=True
rsp = simplejson.load(conn)
f = open('output_lda.txt', 'w')
f1 = open('output_lda_ids.txt','w')
for doc in rsp['response']['docs']:
last_trailing_word = "[^' 'a-zA-Z#:\/0-9._@,]*"
pat = re.compile( last_trailing_word )
f1.write(doc['id'] + "\n")
for s in doc['tweet_tags']:
tweet = pat.sub('',s)
#if initial==True:
f.write(tweet+" ")
#initial=False
#else:
#f.write("\n" + tweet)
f.write('\n')
f.close()
f1.close()
def get_data(self, **kwargs):
if not self.auth:
raise AssertionError("You must authenticate() before making requests.")
# establish defaults for all variables
league = ""
season = ""
feed = ""
output_format = ""
params = {}
# iterate over args and assign vars
for key, value in kwargs.items():
if str(key) == 'league':
league = value
elif str(key) == 'season':
season = value
elif str(key) == 'feed':
feed = value
elif str(key) == 'format':
output_format = value
else:
params[key] = value
# add force=false parameter (helps prevent unnecessary bandwidth use)
if not "force" in params:
params['force'] = 'false'
if self.__verify_feed(feed) == False:
raise ValueError("Unknown feed '" + feed + "'. Known values are: " + str(self.valid_feeds))
if self.__verify_format(output_format) == False:
raise ValueError("Unsupported format '" + output_format + "'.")
url = self.determine_url(league, season, feed, output_format, params)
if self.verbose:
print("Making API request to '{}'.".format(url))
print(" with headers:")
print(self.headers)
print(" and params:")
print(params)
r = requests.get(url, params=params, headers=self.headers)
if r.status_code == 200:
if self.store_type != None:
self.__save_feed(r, league, season, feed, output_format, params)
if output_format == "json":
data = json.loads(r.content)
elif output_format == "xml":
data = str(r.content)
else:
data = r.content.splitlines()
elif r.status_code == 304:
if self.verbose:
print("Data hasn't changed since last call")
filename = self.__make_output_filename(league, season, feed, output_format, params)
with open(self.store_location + filename) as f:
if output_format == "json":
data = json.load(f)
elif output_format == "xml":
data = str(f.readlines()[0])
else:
data = f.read().splitlines()
else:
raise Warning("API call failed with error: {error}".format(error=r.status_code))
return data
def qsub(conf_file):
tpl = """
#! /bin/sh
#? nom du job affiche
#PBS -N @exp
#PBS -o stdout
#PBS -b stderr
#PBS -M @email
# maximum execution time
#PBS -l walltime=@wall_time
# mail parameters
#PBS -m abe
# number of nodes
#PBS -l nodes=@nb_cores:ppn=@ppn
#PBS -l pmem=5200mb -l mem=5200mb
export LD_LIBRARY_PATH=@ld_lib_path
exec @exec
"""
if os.environ.has_key('LD_LIBRARY_PATH'):
ld_lib_path = os.environ['LD_LIBRARY_PATH']
else:
ld_lib_path = "''"
home = os.environ['HOME']
print 'LD_LIBRARY_PATH=' + ld_lib_path
# parse conf
conf = simplejson.load(open(conf_file))
exps = conf['exps']
nb_runs = conf['nb_runs']
res_dir = conf['res_dir']
bin_dir = conf['bin_dir']
wall_time = conf['wall_time']
use_mpi = "false"
try:
use_mpi = conf['use_mpi']
except:
use_mpi = "false"
try:
nb_cores = conf['nb_cores']
except:
nb_cores = 1
try:
args = conf['args']
except:
args = ''
email = conf['email']
if (use_mpi == "true"):
ppn = '1'
mpirun = 'mpirun'
else:
nb_cores = 1
ppn = '8'
mpirun = ''
for i in range(0, nb_runs):
for e in exps:
directory = res_dir + "/" + e + "/exp_" + str(i)
try:
os.makedirs(directory)
except:
print "WARNING, dir:" + directory + " not be created"
subprocess.call('cp ' + bin_dir + '/' + e + ' ' + directory,
shell=True)
fname = home + "/tmp/" + e + "_" + str(i) + ".job"
f = open(fname, "w")
f.write(
tpl.replace("@exp", e).replace("@email", email).replace(
"@ld_lib_path",
ld_lib_path).replace("@wall_time", wall_time).replace(
"@dir",
directory).replace("@nb_cores", str(nb_cores)).replace(
"@ppn", ppn).replace(
"@exec", mpirun + ' ' + directory + '/' + e +
' ' + args))
f.close()
s = "qsub -d " + directory + " " + fname
print "executing:" + s
retcode = subprocess.call(s, shell=True, env=None)
print "qsub returned:" + str(retcode)
def info(self, dircap, path):
path = self.fixwinpath(path, False)
meta = json.load(
self.connection.get(u'/uri/%s%s' % (dircap, path),
{u't': u'json'}))
return self._info(path, meta)
def get_exp(conf_file):
conf = simplejson.load(open(conf_file))
return conf['exp'].sub('exp/', '')
import simplejson, urllib
orig_coord = orig_lat, orig_lng
dest_coord = dest_lat, dest_lng
url = "http://maps.googleapis.com/maps/api/distancematrix/json?origins={0}&destinations={1}&mode=driving&language=en-EN&sensor=false".format(str(orig_coord),str(dest_coord))
result= simplejson.load(urllib.urlopen(url))
driving_time = result['rows'][0]['elements'][0]['duration']['value']
if res[0][0] > 0:
print str(res[0][0]) + " " + pre + " already for user [" + user_id + "]: this script will exit..."
sys.exit()
if user_id > 0:
conn = httplib.HTTPConnection("api.twitter.com")
try:
url = "/1/" + pre + "/ids.json?user_id=" + user_id
if cursor is not None:
url += "&cursor=" + cursor
print "http://api.twitter.com/" + url
conn.request("GET", url)
except (Exception):
sys.exit(sys.exc_info())
r = conn.getresponse()
sql = "INSERT INTO " + t + " (source_id, target_id, retrieved) VALUES (" + str(user_id) + ", %d, NOW()) "
if r.status == 200:
j = simplejson.load(r)
if len(j["ids"]) > 0:
for x in j["ids"]:
#sel = pgconn.query(sql % x)
try:
print pgconn.insert(t, {"source_id": user_id, "target_id": x, "retrieved": "NOW()"})
except pg.ProgrammingError:
print "already exists: " + str(user_id) + "->" + str(x)
continue
if len(j["ids"]) >= 5000:
print j["next_cursor_str"]
else:
print r.read()
def launch_exp(conf_file):
print '--- launch exp ---'
# parse configuration
print 'launch, conf = ' + conf_file
conf = simplejson.load(open(conf_file))
machines = conf['machines']
nb_runs = conf['nb_runs']
exp = conf['exp']
directory = conf['dir']
debug = conf['debug']
args = ""
if 'args' in conf: args = conf['args']
print 'exp = ' + exp
print 'dir = ' + directory
print 'nb_runs = ' + str(nb_runs)
print 'debug = ' + str(debug)
print 'machines =' + str(machines)
print 'args =' + str(args)
# copy binaries (debug and opt) & json file
exe = get_exe(conf_file)
try:
os.makedirs(directory + '/bin')
os.system('cp ' + 'build/default/' + exp + '/' + exe + ' ' +
directory + '/bin/' + exe + '_opt')
os.system('cp ' + 'build/debug/' + exp + '/' + exe + ' ' + directory +
'/bin/' + exe + '_debug')
print conf
print directory
os.system('cp ' + conf_file + ' ' + directory)
# create directories
for i in range(0, nb_runs * len(machines)):
os.makedirs(directory + '/exp_' + str(i))
except:
print '/!\ files exist, I cannot replace them'
return
print 'dirs created'
# make a svn diff
status, version = commands.getstatusoutput('svnversion')
if version[len(version) - 1] == 'M':
version = version[0:len(version) - 1]
os.system('svn diff >' + directory + '/' + version + '.diff')
print 'diff done [version=' + version + ']'
# run on each machines
if debug == 1:
exe = exe + '_debug'
else:
exe = exe + '_opt'
if os.environ.has_key('LD_LIBRARY_PATH'):
ld_lib_path = os.environ['LD_LIBRARY_PATH']
else:
ld_lib_path = "''"
k = 0
pids = []
for m in machines.iterkeys():
pid = os.fork()
if (pid == 0): #son
for i in range(0, machines[m]):
if m == 'localhost':
s = "export LD_LIBRARY_PATH=" + ld_lib_path + \
" && cd " + os.getcwd() + '/' + directory + '/exp_'+ str(i + k) + \
" && " + os.getcwd() + '/' + directory + '/bin/' + exe + " " + args + \
" 1> stdout 2> stderr"
else:
s = "ssh -o CheckHostIP=false " + m + \
" 'export LD_LIBRARY_PATH=" + ld_lib_path + \
" && cd " + os.getcwd() + '/' + directory + '/exp_'+ str(i + k) + \
" && " + os.getcwd() + '/' + directory + '/bin/' + exe + " " + args + \
" 1> stdout 2> stderr'"
print 'run ' + str(i + k) + ' on ' + m
print s
ret = subprocess.call(s, shell=True)
print "ret = " + str(ret)
exit(0)
pids += [pid]
k += machines[m]
print "waitpid..."
for i in pids:
os.waitpid(i, 0)
def load(self):
assert op.exists(self.path), self.path
logging.debug('%s: loading info from: %s' %
(self.__class__.__name__, self.path))
self.info = json.load(open(self.path))
logging.debug(self.dump())
def loadleveler(conf_file):
tpl = """
# @ job_name=<name>
# @ output = $(job_name).$(jobid)
# @ error = $(output)
# @ job_type = serial
# @ class = <class>
# @ resources=ConsumableMemory(<memory>) ConsumableCpus(<cpu>)
# @ queue
export LD_LIBRARY_PATH=<ld_library_path>
cd <initial_dir>
./<exec>
"""
if os.environ.has_key('LD_LIBRARY_PATH'):
ld_lib_path = os.environ['LD_LIBRARY_PATH']
else:
ld_lib_path = "''"
home = os.environ['HOME']
print 'LD_LIBRARY_PATH=' + ld_lib_path
# parse conf
conf = simplejson.load(open(conf_file))
jobname = conf['jobname']
exps = conf['exps']
nb_runs = conf['nb_runs']
res_dir = conf['res_dir']
bin_dir = conf['bin_dir']
jobclass = conf['class']
try:
memory = conf['memory']
except:
memory = 3000
try:
cpu = conf['cpu']
except:
cpu = 1
for i in range(0, nb_runs):
for e in exps:
directory = res_dir + "/" + e + "/exp_" + str(i)
try:
os.makedirs(directory)
except:
print "WARNING, dir:" + directory + " cannot be created"
subprocess.call('cp ' + bin_dir + '/' + e + ' ' + directory,
shell=True)
try:
os.makedirs(home + "/tmp")
except:
pass
fname = home + "/tmp/" + e + "_" + str(i) + ".job"
f = open(fname, "w")
f.write(
tpl.replace("<name>", jobname).replace(
"<ld_library_path>",
ld_lib_path).replace("<class>", jobclass).replace(
"<initial_dir>",
directory).replace("<memory>", str(memory)).replace(
"<cpu>", str(cpu)).replace("<exec>", e))
f.close()
s = "llsubmit " + fname
print "executing:" + s
retcode = subprocess.call(s, shell=True, env=None)
print "llsubmit returned:" + str(retcode)
def insert(self, h=None):
if h:
return json.dumps(self.cluster.insert(int(h)))
return json.dumps(self.cluster.insert(*json.load(request.body)))
def newFromJsonFile(jsonfile):
with open(jsonfile) as json_file:
json_data = json.load(json_file)
return LimitRanges.newFromDict(json_data)
def metric_init(params):
descriptors = []
logging.debug('[elasticsearch] Received the following parameters')
logging.debug(params)
host = params.get('host', 'http://localhost:9200/')
version = params.get('version', '1.3')
m = re.match('(?P<major>\d+)\.(?P<minor>(\d+(\.\d+)*))', version)
if m and m.group('major') == '0':
url_cluster = '{0}_cluster/nodes/_local/stats?all=true'.format(host)
else:
url_cluster = '{0}_nodes/_local/stats'.format(host)
# First iteration - Grab statistics
logging.debug('[elasticsearch] Fetching ' + url_cluster)
result = json.load(urllib.urlopen(url_cluster))
metric_group = params.get('metric_group', 'elasticsearch')
Desc_Skel = {
'name': 'XXX',
'call_back': partial(getStat, result, url_cluster),
'time_max': 60,
'value_type': 'uint',
'units': 'units',
'slope': 'both',
'format': '%d',
'description': 'XXX',
'groups': metric_group,
}
indices = params.get('indices', '*').split()
for index in indices:
url_indices = '{0}{1}/_stats'.format(host, index)
logging.debug('[elasticsearch] Fetching ' + url_indices)
r_indices = json.load(urllib.urlopen(url_indices))
descriptors += get_indices_descriptors(index, Desc_Skel, r_indices,
url_indices)
_create_desc = partial(create_desc, Desc_Skel)
descriptors.append(
_create_desc({
'name': 'es_heap_committed',
'units': 'Bytes',
'format': '%.0f',
'description': 'Java Heap Committed (Bytes)',
'value_type': 'double'
}))
descriptors.append(
_create_desc({
'name': 'es_heap_used',
'units': 'Bytes',
'format': '%.0f',
'description': 'Java Heap Used (Bytes)',
'value_type': 'double'
}))
descriptors.append(
_create_desc({
'name': 'es_non_heap_committed',
'units': 'Bytes',
'format': '%.0f',
'description': 'Java Non Heap Committed (Bytes)',
'value_type': 'double'
}))
descriptors.append(
_create_desc({
'name': 'es_non_heap_used',
'units': 'Bytes',
'format': '%.0f',
'description': 'Java Non Heap Used (Bytes)',
'value_type': 'double'
}))
descriptors.append(
_create_desc({
'name': 'es_threads',
'units': 'threads',
'format': '%d',
'description': 'Threads (open)',
}))
descriptors.append(
_create_desc({
'name': 'es_threads_peak',
'units': 'threads',
'format': '%d',
'description': 'Threads Peak (open)',
}))
descriptors.append(
_create_desc({
'name': 'es_gc_young_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Java GC Young Time (ms)'
}))
descriptors.append(
_create_desc({
'name': 'es_gc_old_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Java GC Old Time (ms)'
}))
descriptors.append(
_create_desc({
'name': 'es_transport_open',
'units': 'sockets',
'format': '%d',
'description': 'Transport Open (sockets)',
}))
descriptors.append(
_create_desc({
'name': 'es_transport_rx_count',
'units': 'rx',
'format': '%d',
'slope': 'positive',
'description': 'RX Count'
}))
descriptors.append(
_create_desc({
'name': 'es_transport_rx_size',
'units': 'Bytes',
'format': '%.0f',
'slope': 'positive',
'description': 'RX (Bytes)',
'value_type': 'double',
}))
descriptors.append(
_create_desc({
'name': 'es_transport_tx_count',
'units': 'tx',
'format': '%d',
'slope': 'positive',
'description': 'TX Count'
}))
descriptors.append(
_create_desc({
'name': 'es_transport_tx_size',
'units': 'Bytes',
'format': '%.0f',
'slope': 'positive',
'description': 'TX (Bytes)',
}))
descriptors.append(
_create_desc({
'name': 'es_http_current_open',
'units': 'sockets',
'format': '%d',
'description': 'HTTP Open (sockets)',
}))
descriptors.append(
_create_desc({
'name': 'es_http_total_open',
'units': 'sockets',
'format': '%d',
'description': 'HTTP Open (sockets)',
}))
descriptors.append(
_create_desc({
'name': 'es_indices_size',
'units': 'Bytes',
'format': '%.0f',
'description': 'Index Size (Bytes)',
'value_type': 'double',
}))
descriptors.append(
_create_desc({
'name': 'es_gc_young_count',
'format': '%d',
'slope': 'positive',
'description': 'Java GC Young Count',
}))
descriptors.append(
_create_desc({
'name': 'es_gc_old_count',
'format': '%d',
'slope': 'positive',
'description': 'Java GC Old Count',
}))
descriptors.append(
_create_desc({
'name': 'es_merges_current',
'format': '%d',
'description': 'Merges (current)',
}))
descriptors.append(
_create_desc({
'name': 'es_merges_current_docs',
'format': '%d',
'description': 'Merges (docs)',
}))
descriptors.append(
_create_desc({
'name': 'es_merges_total',
'format': '%d',
'slope': 'positive',
'description': 'Merges (total)',
}))
descriptors.append(
_create_desc({
'name': 'es_merges_total_docs',
'format': '%d',
'slope': 'positive',
'description': 'Merges (total docs)',
}))
descriptors.append(
_create_desc({
'name': 'es_merges_current_size',
'units': 'Bytes',
'format': '%.0f',
'description': 'Merges size (current)',
}))
descriptors.append(
_create_desc({
'name': 'es_merges_total_size',
'units': 'Bytes',
'format': '%.0f',
'slope': 'positive',
'description': 'Merges size (total)',
'value_type': 'double',
}))
descriptors.append(
_create_desc({
'name': 'es_merges_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Merges Time (ms)'
}))
descriptors.append(
_create_desc({
'name': 'es_refresh_total',
'units': 'refreshes',
'format': '%d',
'slope': 'positive',
'description': 'Total Refresh'
}))
descriptors.append(
_create_desc({
'name': 'es_refresh_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Total Refresh Time'
}))
descriptors.append(
_create_desc({
'name': 'es_docs_count',
'units': 'docs',
'format': '%.0f',
'description': 'Number of Documents',
'value_type': 'double'
}))
descriptors.append(
_create_desc({
'name': 'es_docs_deleted',
'units': 'docs',
'format': '%.0f',
'description': 'Number of Documents Deleted',
'value_type': 'double'
}))
descriptors.append(
_create_desc({
'name': 'es_open_file_descriptors',
'units': 'files',
'format': '%d',
'description': 'Open File Descriptors',
}))
descriptors.append(
_create_desc({
'name': 'es_cache_fielddata_evictions',
'units': 'units',
'format': '%d',
'slope': 'positive',
'description': 'Field Data Cache Evictions',
}))
descriptors.append(
_create_desc({
'name': 'es_cache_fielddata_size',
'units': 'Bytes',
'format': '%.0f',
'description': 'Field Data Cache Size',
'value_type': 'double',
}))
descriptors.append(
_create_desc({
'name': 'es_cache_filter_evictions',
'format': '%d',
'slope': 'positive',
'description': 'Filter Cache Evictions',
}))
descriptors.append(
_create_desc({
'name': 'es_cache_filter_size',
'units': 'Bytes',
'format': '%.0f',
'description': 'Filter Cache Size',
'value_type': 'double'
}))
descriptors.append(
_create_desc({
'name': 'es_query_current',
'units': 'Queries',
'format': '%d',
'description': 'Current Queries',
}))
descriptors.append(
_create_desc({
'name': 'es_query_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Total Query Time'
}))
descriptors.append(
_create_desc({
'name': 'es_fetch_current',
'units': 'fetches',
'format': '%d',
'description': 'Current Fetches',
}))
descriptors.append(
_create_desc({
'name': 'es_fetch_total',
'units': 'fetches',
'format': '%d',
'slope': 'positive',
'description': 'Total Fetches'
}))
descriptors.append(
_create_desc({
'name': 'es_fetch_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Total Fetch Time'
}))
descriptors.append(
_create_desc({
'name': 'es_flush_total',
'units': 'flushes',
'format': '%d',
'description': 'Total Flushes',
}))
descriptors.append(
_create_desc({
'name': 'es_flush_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Total Flush Time'
}))
descriptors.append(
_create_desc({
'name': 'es_get_exists_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Exists Time'
}))
descriptors.append(
_create_desc({
'name': 'es_get_exists_total',
'units': 'total',
'format': '%d',
'description': 'Exists Total',
}))
descriptors.append(
_create_desc({
'name': 'es_get_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Get Time'
}))
descriptors.append(
_create_desc({
'name': 'es_get_total',
'units': 'total',
'format': '%d',
'description': 'Get Total',
}))
descriptors.append(
_create_desc({
'name': 'es_get_missing_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Missing Time'
}))
descriptors.append(
_create_desc({
'name': 'es_get_missing_total',
'units': 'total',
'format': '%d',
'description': 'Missing Total',
}))
descriptors.append(
_create_desc({
'name': 'es_indexing_delete_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Delete Time'
}))
descriptors.append(
_create_desc({
'name': 'es_indexing_delete_total',
'units': 'docs',
'format': '%d',
'slope': 'positive',
'description': 'Delete Total'
}))
descriptors.append(
_create_desc({
'name': 'es_indexing_index_time',
'units': 'ms',
'format': '%d',
'slope': 'positive',
'description': 'Indexing Time'
}))
descriptors.append(
_create_desc({
'name': 'es_indexing_index_total',
'units': 'docs',
'format': '%d',
'slope': 'positive',
'description': 'Indexing Documents Total'
}))
descriptors.append(
_create_desc({
'name': 'es_query_total',
'units': 'Queries',
'format': '%d',
'slope': 'positive',
'description': 'Total Queries'
}))
return descriptors
def remove(self, h=None):
if h:
return json.dumps(self.cluster.remove(int(h)))
return json.dumps(self.cluster.remove(*json.load(request.body)))
def setUp(self):
with open("sensor_data_network_config.json", "rb") as jsonFile:
self.templateNetworkConfig = json.load(jsonFile)
def all(self, query=None):
if query:
return json.dumps(self.cluster.find_all(int(query)))
return json.dumps(self.cluster.find_all(*json.load(request.body)))
def maybe_get(x):
return x
if __name__ == "__main__":
random.seed(435)
np.random.seed(2342)
np.set_printoptions(linewidth=100)
data_dir = os.path.join("../data", "stanfordmovie")
with open(
os.path.join(data_dir,
"stanfordmovie.train.sentences.clean.projected.json")
) as data_file:
data = json.load(data_file)
random.shuffle(data)
X, Y = map(list, zip(*data))
Y = [[":)", ":("].index(y) for y in Y]
with open(
os.path.join(
data_dir,
"stanfordmovie.train.sentences.clean.dictionary.encoding.json")
) as encoding_file:
encoding = json.load(encoding_file)
print len(encoding)
n_validation = 500
batch_size = 25
def update_result(result, url):
logging.debug('[elasticsearch] Fetching ' + url)
result = json.load(urllib.urlopen(url))
return result
# Standard #
############
import os
import shutil
from pathlib import Path
###############
# Third Party #
###############
import pytest
import simplejson as sjson
from cookiecutter import main
CCDSA_ROOT = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir))
TEMPLATE_DIR = Path(CCDSA_ROOT) / "{{ cookiecutter.folder_name }}"
CC_JSON = sjson.load(open(str(Path(CCDSA_ROOT) / "cookiecutter.json")))
@pytest.fixture(scope='function')
def default_baked_project(tmpdir):
out_dir = str(tmpdir.mkdir('data-project'))
main.cookiecutter(CCDSA_ROOT,
no_input=True,
extra_context={},
output_dir=out_dir)
# default project name is project_name
yield os.path.join(out_dir, 'project_name')
# cleanup after
c.execute("CREATE TABLE IF NOT EXISTS users(id integer PRIMARY KEY, username text, phone text, website text, regdate text)") # AUTOINCREMENT : 자동 증가, PRIMARY KEY : 기본 키. 중복이 될 수 없는 값
# 데이터 삽입
c.execute('INSERT INTO users VALUES(1, "kim", "010-000-0000", "kim.co.kr", ?)', (nowDateTime,)) # 매핑은 튜플로 한다.
userList = (
(2, "kim", "010-000-0000", "kim.co.kr", nowDateTime),
(3, "park", "010-000-0000", "park.co.kr", nowDateTime),
(4, "lee", "010-000-0000", "lee.co.kr", nowDateTime),
)
# c.executemany("INSERT INTO users(id, username, phone, website, regdate) VALUES(?,?,?,?,?)", userList)
with open('data/users.json', 'r') as infile :
r = json.load(infile)
userData = []
for user in r :
tup = (user['id'], user['name'], user['phone'], user['website'], nowDateTime)
# print(tup)
userData.append(tup)
c.execute("INSERT INTO users VALUES(?,?,?,?,?)", tup)
# c.executemany("INSERT INTO users(id, username, phone, website, regdate) VALUES(?,?,?,?,?)", userData) # userData리스트를 알아서 튜플로 형변환 해 준다.
# 데이터 삭제
# print('users db delete', conn.execute('delete from users').rowcount, "rows")
conn.commit()
