def prepare_env(cnt, file_name=True, inline=False): this = {} if file_name: this['file_name_json'] = os.path.join(SAMPLES_PATH, '{}.json'.format(cnt)) this['file_name_rst'] = os.path.join(SAMPLES_PATH, '{}{inline}.rst'.format(cnt, inline='_inline' if inline else '') ) with codecs.open(this['file_name_json'], 'r', encoding='utf-8') as _file: doc = json.load(_file) else: this['file_name_json'] = False this['file_name_rst'] = False doc = json.load(cnt) this['swagger_doc'] = rst.SwaggerObject(doc) doc_module = importlib.import_module('swg2rst.utils.rst') jinja_env = Environment(lstrip_blocks=True, trim_blocks=True) jinja_env.loader = PackageLoader('swg2rst') for name, function in inspect.getmembers(doc_module, inspect.isfunction): jinja_env.filters[name] = function jinja_env.filters['sorted'] = sorted template = jinja_env.get_template('main.rst') this['raw_rst'] = template.render(doc=this['swagger_doc'], inline=inline) this['pattern'] = re.compile(r'[idm]_\w{32}') this['normalize'] = lambda x: x[:-1] if x[-1] == '\n' else x return this
def query_api(path, limit=None, **kwargs): """Returns an iterator over the objects returned from the govtrack.us API. All additional keyword paramaters are passed onto the govtrack.us API in the form of key/value pairs in the querystring of the URL. For further information on the parameters available see the govtrack.us API docs at http://www.govtrack.us/developers/api. Arguments: path -- the filepath of the API endpoint to query Keyword Arguments: limit -- the number of objects to return. If None (default), all objects are returned """ url = GovTrackURL(path=path, **kwargs) while True: response = json.load(urllib2.urlopen(url.to_string())) meta = response['meta'] objects = response['objects'] offset = meta['offset'] limit = limit if limit is not None else meta['total_count'] for obj in objects: offset += 1 progress = int((float(offset)/limit) * 100) obj_url = GovTrackURL(path='%s/%s' % (path, obj['id'])) yield json.load(urllib2.urlopen(obj_url.to_string())), progress # If the limit argument is set, exit after we've reached that limit if limit is not None and offset >= limit: return # Update the current offset value and, if we've retrieved all of the # objects available, exit this function url.offset = meta['offset'] + meta['limit']
def reverse_geocode(self, lon, lat): params = { 'api_key': settings.API_KEYS['cloudmade'], 'lon': lon, 'lat': lat, 'type': 'road', } data = urllib2.urlopen(self.REVERSE_GEOCODE_URL % params) json = simplejson.load(data) if not json: return [] else: name = json['features'][0]['properties'].get('name') try: params['type'] = 'area' data = simplejson.load(urllib2.urlopen(self.REVERSE_GEOCODE_URL % params)) name = '%s, %s' % (name, data['features'][0]['properties']['name']) except Exception: pass return [{ 'name': json['features'][0]['properties'].get('name'), 'location': (lon, lat), 'accuracy': 100, }]
def POST(self, id=None, outputs=None, **kwargs): if id is None: metadata = self.parse_json_dict(simplejson.load(cherrypy.request.body)) for inp in metadata.get("inputs",[]): if '_id' in inp: inp['_id'] = self.parse_value(inp['_id']) id = Transformations.insert_one(metadata).inserted_id # for inp in metadata.get("inputs",[]): # Products.update_one({'_id':inp['_id']},{'$push':{'transformations':{'_id':id}}}) return self.build_response('ok', **dict(transformation={'_id':id})) else: if outputs == 'outputs': metadata = self.parse_json_dict(simplejson.load(cherrypy.request.body)) resp = simplejson.loads(ProductsController().POST(id=None, metadata=metadata)) if resp['status'] != 'ok': return self.build_response('error', message='error creating output product: %s' % resp['message']) Transformations.update_one({'_id':ObjectId(id)},{'$push':{'outputs':{'_id':ObjectId(resp['data']['product']['_id'])}}}) return self.build_response('ok', **dict(product=resp['data']['product'])) else: return self.build_response('error',message='cannot POST metadata for an existing transformation. Use PUT instead.')
def generate(mapping, schema, filename): mapping = json.load(open(mapping)) schema = json.load(open(schema)) schema.pop('$schema', 0) # Index for which keys need to be marked as having exact counterparts exact_index = get_exact_fields(mapping) # Index to lookup a non-array version of key name (so it maps to ES) es_items_index = generate_es_to_schema_index(schema) # All the fields to process from the schema schema_fields = get_schema_fields(schema) for field in schema_fields: type_field = field + '.' + 'type' # No need expand these complex types with DEFAULT attributes if get_deep(schema, type_field) in ['object', 'array']: continue if not field: continue es_field = es_items_index.get(field, None) is_exact = True if es_field in exact_index else False # Expand each of the keys to match DEFAULTS. # set_deep() will not overwrite data if it does exist, but will add if it # does not for key, value in DEFAULTS.items(): full_key = field + '.' + key if is_exact and key == 'is_exact': value = True set_deep(schema, full_key, value) make_yaml(schema, filename)
def load(): states = simplejson.load(file(DATA_DIR + '/states/index.json')) for code, state in states.iteritems(): if 'aka' in state: state.pop('aka') db.insert('state', seqname=False, code=code, **unidecode(state)) districts = simplejson.load(file(DATA_DIR + '/districts/index.json')) for name, district in districts.iteritems(): db.insert('district', seqname=False, name=name, **unidecode(district)) for fn in ['almanac', 'shapes', 'centers']: print 'loading', fn districts = simplejson.load(file(DATA_DIR + '/districts/%s.json' % fn)) for name, district in districts.iteritems(): db.update('district', where='name = $name', vars=locals(), **unidecode(district)) politicians = simplejson.load(file(DATA_DIR + '/politicians/index.json')) for polid, pol in politicians.iteritems(): db.insert('politician', seqname=False, id=polid, **unidecode(pol)) for fn in ['govtrack', 'photos']: print 'loading', fn politicians = simplejson.load(file(DATA_DIR + '/politicians/%s.json' % fn)) for polid, pol in politicians.iteritems(): db.update('politician', where='id = $polid', vars=locals(), **unidecode(pol))
def index(request): conn = urlopen( settings.RUNMODO_SOLR_URL + '/select?group=true&group.field=eventName&sort=raceStart%20desc&fq=raceStart:[*%20TO%20NOW]&rows=5') results = simplejson.load(conn) conn = urlopen( settings.RUNMODO_SOLR_URL + '/select?group=true&group.field=eventName&sort=raceStart%20asc&fq=raceStart:[NOW%20TO%20*]&rows=5') events = simplejson.load(conn) return render_to_response('runapp/index2.html', {'results': results['grouped']['eventName']['groups'], 'events': events['grouped']['eventName']['groups'], 'view':'index'}, context_instance=RequestContext(request) )
def _resume(results_dir): """Loads a partially completed json results directory.""" # Pylint can't infer that the json being loaded is a dict # pylint: disable=maybe-no-member assert os.path.isdir(results_dir), \ "TestrunResult.resume() requires a directory" # Load the metadata with open(os.path.join(results_dir, 'metadata.json'), 'r') as f: meta = json.load(f) assert meta['results_version'] == CURRENT_JSON_VERSION, \ "Old results version, resume impossible" testrun = results.TestrunResult() testrun.name = meta['name'] testrun.options = meta['options'] testrun.uname = meta.get('uname') testrun.glxinfo = meta.get('glxinfo') testrun.lspci = meta.get('lspci') # Load all of the test names and added them to the test list for file_ in os.listdir(os.path.join(results_dir, 'tests')): with open(os.path.join(results_dir, 'tests', file_), 'r') as f: try: testrun.tests.update(json.load(f, object_hook=piglit_decoder)) except ValueError: continue return testrun
def _resume(results_dir): """Loads a partially completed json results directory.""" # Pylint can't infer that the json being loaded is a dict # pylint: disable=maybe-no-member assert os.path.isdir(results_dir), \ "TestrunResult.resume() requires a directory" # Load the metadata with open(os.path.join(results_dir, 'metadata.json'), 'r') as f: meta = json.load(f) assert meta['results_version'] == CURRENT_JSON_VERSION, \ "Old results version, resume impossible" meta['tests'] = collections.OrderedDict() # Load all of the test names and added them to the test list tests_dir = os.path.join(results_dir, 'tests') file_list = sorted( (l for l in os.listdir(tests_dir) if l.endswith('.json')), key=lambda p: int(os.path.splitext(p)[0])) for file_ in file_list: with open(os.path.join(tests_dir, file_), 'r') as f: try: meta['tests'].update(json.load(f)) except ValueError: continue return results.TestrunResult.from_dict(meta)
def suckdry_planlos(): import urllib f = urllib.urlopen("http://planlosbremen.de/termine/service/monat") #https://planlosbremen.de/termine/service/location/5 #jsondata = f.read() data = simplejson.load(f) u = db.User.find_one() for i in data: loc_id = i['fields']['location'] url = urllib.urlopen("http://planlosbremen.de/termine/service/location/%s" % loc_id) loc = simplejson.load(url) location = loc_name = loc[0]['fields']['name'] e = db.Event() f = i['fields'] e.title = unicode(f['title']) e.author = u form_date = date_parser(f['datum']).date() form_time = date_parser(f['time']).time() e.eventdate = datetime.combine(form_date, form_time) e.short_desc = unicode(f['short_desc']) e.desc = unicode(f['desc']) e.url = unicode(f['exturl']) e.tags = [unicode(f['type'])] e.is_published = f['is_pub'] e.save()
def run_compare(): with open('url-content-read.json') as f: readability_text = simplejson.load(f) with open('url-content-fetcher.json') as f: fetcher_text = simplejson.load(f) cnt = 0 z_cnt = 0 dmp = diff_match_patch() rets = [] for key, value in readability_text.items(): if key in fetcher_text: cnt += 1 rc = re.sub(r' ', '', value) fc = re.sub(r' ', '', fetcher_text[key]) l_len = len(rc) r_len = len(fc) retval = dif_content(rc, fc) retval_ground = 0 results = dmp.diff_main(rc, fc) for res in results: if res[0] == 0: retval_ground += len(res[1]) print cnt, ': ', l_len, r_len, retval, retval_ground real_ret = max(retval, retval_ground) rets.append((cnt, l_len, r_len, real_ret)) with open('diff_result_1', 'w') as f: for res in rets: print >> f, res[0], ': ', res[1], res[2], res[3]
def test_jsonsink(): import simplejson import os ds = nio.JSONFileSink() yield assert_equal, ds.inputs._outputs, {} ds = nio.JSONFileSink(in_dict={'foo': 'var'}) yield assert_equal, ds.inputs.in_dict, {'foo': 'var'} ds = nio.JSONFileSink(infields=['test']) yield assert_true, 'test' in ds.inputs.copyable_trait_names() curdir = os.getcwd() outdir = mkdtemp() os.chdir(outdir) js = nio.JSONFileSink(infields=['test'], in_dict={'foo': 'var'}) js.inputs.new_entry = 'someValue' setattr(js.inputs, 'contrasts.alt', 'someNestedValue') res = js.run() with open(res.outputs.out_file, 'r') as f: data = simplejson.load(f) yield assert_true, data == {"contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"} js = nio.JSONFileSink(infields=['test'], in_dict={'foo': 'var'}) js.inputs.new_entry = 'someValue' js.inputs.test = 'testInfields' setattr(js.inputs, 'contrasts.alt', 'someNestedValue') res = js.run() with open(res.outputs.out_file, 'r') as f: data = simplejson.load(f) yield assert_true, data == {"test": "testInfields", "contrasts": {"alt": "someNestedValue"}, "foo": "var", "new_entry": "someValue"} os.chdir(curdir) shutil.rmtree(outdir)
def query(self, object_id=None,type=None,connection_type=None, metadata=False): response = {} import urllib import simplejson if type == 'fb': url = 'https://graph.facebook.com/%s' % (object_id) if connection_type: url += '/%s' % (connection_type) params = {'access_token': self.access_token} if metadata: params['metadata'] = 1 url += '?' + urllib.urlencode(params) response = simplejson.load(urllib.urlopen(url)) if 'error' in response: error = response['error'] raise FacebookSessionError(error['type'], error['message']) if type == 'github': url = 'https://api.github.com/user' params = {'access_token': self.access_token} url += '?' + urllib.urlencode(params) print "DAY LA URL GITHUB %s " % url response = simplejson.load(urllib.urlopen(url)) return response
def update(self): request = self.request id = request.model_id fs = self.get_fieldset(suffix="Edit", id=id) event = events.BeforeRenderEvent(self.request.model_instance, self.request, fs=fs) alsoProvides(event, events.IBeforeEditRenderEvent) zope.component.event.objectEventNotify(event) if request.format == "json" and request.method == "PUT": data = json.load(request.body_file) elif request.content_type == "application/json": data = json.load(request.body_file) else: data = request.POST with_prefix = True if request.format == "json": with_prefix = bool(request.params.get("with_prefix")) fs = fs.bind(request=request, with_prefix=with_prefix) if self.validate(fs): fs.sync() self.sync(fs, id) self.session.flush() if request.format in ("html", "xhr"): if request.is_xhr or request.format == "xhr": return Response(content_type="text/plain") return exc.HTTPFound(location=request.fa_url(request.model_name, _pk(fs.model))) else: return self.render(fs=fs, status=0) if request.format == "html": return self.render(fs=fs, id=id) else: return self.render(fs=fs, status=1)
def read_config(self): try: logger.info("going to read the sites config file ") with open(NewsCrawlerConfig.SITE_FILE, "rbU") as config: try: logger.info("going to json load the sites config") json.load(config, object_hook = self.as_newscrawler) self.is_config_read = True except Exception as load_e: logger.error(load_e) raise load_e except IOError as file_e: logger.error(file_e) raise file_e try: logger.info("going to read the crawler configurations") with open(NewsCrawlerConfig.CRAWL_FILE, "rbU") as config: try: logger.info("going to json load the crawler configs") json.load(config, object_hook = self.as_crawloptions) except Exception as load_e: logger.error(load_e) raise load_e except IOError as file_e: logger.error(file_e) raise file_e
def searchImg(query): ## reference: https://developers.google.com/image-search/v1/jsondevguide#json_snippets_python query = string.replace(query," ","%20") url = ("https://ajax.googleapis.com/ajax/services/search/images?" + "v=1.0&q=" + query + "&rsz=8") request = urllib2.Request(url, None, {'Referer':"www.github.com/gradiuscypher/Neo"}) response = urllib2.urlopen(request) results = simplejson.load(response) ## list of result pages, used for &start query pagelist = results['responseData']['cursor']['pages'] pages = [] for i in pagelist: pages.append(i['start']) ##return all results, save them to a dictionary, append that dictionary to a list resultList = [] for i in pages: query = string.replace(query," ","%20") url = ("https://ajax.googleapis.com/ajax/services/search/images?" + "v=1.0&q=" + query + "&rsz=8&start=" + i) request = urllib2.Request(url, None, {'Referer':"www.github.com/gradiuscypher/Neo"}) response = urllib2.urlopen(request) results = simplejson.load(response) anslist = results['responseData']['results'] for i in anslist: resultList.append({'name':i['visibleUrl'],'url':i['url'],'titleNoFormatting':i['titleNoFormatting']}) return resultList
def getPages(): """ For each page of the diary: get the json and save in ./Diaries/diaryID/pageID.json """ path = "Diaries/" urlBase = "http://transcripts.sl.nsw.gov.au/api/node/" errorsList = [] for diaryUrl in diariesUrls: print diaryUrl response = urllib2.urlopen(diaryUrl) # print response.info() myjson = simplejson.load(response) pagesList = myjson["field_transcript_pages"]["und"] ## Create folder if it doesn't exist directory = path + diaryUrl[42:-5] if not os.path.exists(directory): os.makedirs(directory) ## Get each page in json for page in pagesList: print page["nid"] mypage = urlBase + page["nid"] + ".json" try: response = urllib2.urlopen(mypage) # print response.info() myjson = simplejson.load(response) file = open(directory + "/" + page["nid"] + ".json", "w+") file.write(str(myjson)) file.close() except urllib2.HTTPError, e: print "We failed with error code - %s." % e.code print mypage print "$$$$$$$$$$$$" errorsList.append("diary:" + diaryUrl[42:-5] + ": page " + mypage)
def test_json_is_valid(self): """ JSONBackend.write_test() produces valid json """ with open(os.path.join(self.tdir, 'tests', '0.json'), 'r') as f: try: json.load(f) except Exception as e: raise AssertionError(e)
def test_results_valid(self): """ JSONBackend.finalize() results.json is valid """ with open(os.path.join(self.tdir, 'results.json'), 'r') as f: try: json.load(f) except Exception as e: raise AssertionError(e)
def _resume(results_dir): """Loads a partially completed json results directory.""" # TODO: could probably use TestrunResult.from_dict here # Pylint can't infer that the json being loaded is a dict # pylint: disable=maybe-no-member assert os.path.isdir(results_dir), \ "TestrunResult.resume() requires a directory" # Load the metadata with open(os.path.join(results_dir, 'metadata.json'), 'r') as f: meta = json.load(f) assert meta['results_version'] == CURRENT_JSON_VERSION, \ "Old results version, resume impossible" meta['tests'] = {} # Load all of the test names and added them to the test list for file_ in os.listdir(os.path.join(results_dir, 'tests')): with open(os.path.join(results_dir, 'tests', file_), 'r') as f: try: meta['tests'].update(json.load(f, object_hook=piglit_decoder)) except ValueError: continue return results.TestrunResult.from_dict(meta)
def test_payload_splitter(self): with open(FIXTURE_PATH + '/legacy_payload.json') as f: legacy_payload = json.load(f) legacy_payload_split, metrics_payload, checkruns_payload = split_payload(dict(legacy_payload)) series = metrics_payload['series'] legacy_payload_split['metrics'] = [] for s in series: attributes = {} if s.get('type'): attributes['type'] = s['type'] if s.get('host'): attributes['hostname'] = s['host'] if s.get('tags'): attributes['tags'] = s['tags'] if s.get('device'): attributes['device_name'] = s['device'] formatted_sample = [s['metric'], s['points'][0][0], s['points'][0][1], attributes] legacy_payload_split['metrics'].append(formatted_sample) del legacy_payload['service_checks'] self.assertEqual(legacy_payload, legacy_payload_split) with open(FIXTURE_PATH + '/sc_payload.json') as f: expected_sc_payload = json.load(f) self.assertEqual(checkruns_payload, expected_sc_payload)
def load_data(dump_file): client = pymongo.MongoClient("localhost", 27017) db = client['courses'] coursesd = db['coursesd'] sectionsd = db['sectionsd'] doc_queue = [] with open(dump_file) as f: for course in json.load(f): pairs = [(name, _typify(course.get(name), data_type)) for (name, data_type) in course_schema if name not in special_fields] pairs += _special_treatment(course, course_schema) doc = dict(pairs) coursesd.update({'CourseFull': doc['CourseFull']}, doc, True) print '%d courses in db.' % coursesd.count() with open(dump_file) as f: for course in json.load(f): pairs = [(name, _typify(course[name], data_type)) for (name, data_type) in section_schema if name not in special_fields] pairs += _special_treatment(course, section_schema) doc = dict(pairs) doc_queue.append(doc) if len(doc_queue) == 100: inserted = sectionsd.insert(doc_queue) print '%d documents inserted' % len(inserted) doc_queue = [] if doc_queue: inserted = sectionsd.insert(doc_queue) print '%d documents inserted' % len(inserted) doc_queue = [] print '%d sections in db.' % sectionsd.count()
def __init__(self, authtype = "OAuth", username = None, password = None, oauth_keys = None, headers = None): self.authtype = authtype self.authenticated = False self.username = username self.password = password self.oauth_keys = oauth_keys if self.username is not None and self.password is not None: if self.authtype == "OAuth": self.request_token_url = 'https://twitter.com/oauth/request_token' self.access_token_url = 'https://twitter.com/oauth/access_token' self.authorization_url = 'http://twitter.com/oauth/authorize' self.signin_url = 'http://twitter.com/oauth/authenticate' # Do OAuth type stuff here - how should this be handled? Seems like a framework question... elif self.authtype == "Basic": self.auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() self.auth_manager.add_password(None, "http://twitter.com", self.username, self.password) self.handler = urllib2.HTTPBasicAuthHandler(self.auth_manager) self.opener = urllib2.build_opener(self.handler) if headers is not None: self.opener.addheaders = [('User-agent', headers)] try: simplejson.load(self.opener.open("http://twitter.com/account/verify_credentials.json")) self.authenticated = True except HTTPError, e: raise TangoError("Authentication failed with your provided credentials. Try again? (%s failure)" % `e.code`, e.code)
def BestMatch(sname, api_code): url = "https://opencorporates.com/reconcile/?query="+urllib.quote(sname) entities = simplejson.load(urllib.urlopen(url)) try: bm = entities['result'][0] url = "https://api.opencorporates.com" + bm['id'] + '?api_token='+ api_code match = simplejson.load(urllib.urlopen(url)) m = match['results']['company'] if m['controlling_entity'] == None: cen = '' cec = '' ceu = '' else: cen = m['controlling_entity']['name'] cec = m['controlling_entity']['jurisdiction_code'] ceu = m['controlling_entity']['opencorporates_url'] print sname, m['name'], m['jurisdiction_code'], m['company_number'], m['corporate_groupings'], m['agent_name'], m['agent_address'], m['alternative_names'], m['previous_names'], m['home_company'], cen, cec, ceu, m['inactive'], bm['score'], bm['match'], bm['uri'], m['registry_url'] reconciled.writerow([sname, m['name'].encode('utf-8'), m['jurisdiction_code'], m['company_number'], m['corporate_groupings'], m['agent_name'], m['agent_address'], m['alternative_names'], m['previous_names'], m['home_company'], cen, cec, ceu, m['inactive'], bm['score'], bm['match'], bm['uri'], m['registry_url']]) return match['results']['company']['name'] except IndexError: reconciled.writerow([sname, 'nomatch']) return "nomatch"
def tz(phenny, input): """Displays time in different time zones""" with open('nickloc.csv', 'rU') as f: z = csv.reader(f) nickdict = {} for key, val in z: nickdict[key] = val try: locinput = input.group(2) locinput1 = locinput.strip().lower() if locinput1 in nickdict: htmlinput = urllib.quote(nickdict[locinput1]) else: htmlinput = urllib.quote(locinput1) url2 = 'http://nominatim.openstreetmap.org/search?q=' + htmlinput + '&format=json' jsonResponse = simplejson.load(urllib.urlopen(url2)) lati = jsonResponse[0]['lat'] longi = jsonResponse[0]['lon'] loca = jsonResponse[0]['display_name'] url3 = 'http://api.geonames.org/timezoneJSON?lat=' + lati + '&lng=' + longi + '&username=jinpark' jsonResponse1 = simplejson.load(urllib.urlopen(url3)) if jsonResponse1['dstOffset'] == 0: timezone = '' elif jsonResponse1['dstOffset'] > 0: timezone = '+' + str(jsonResponse1['dstOffset']) else: timezone = str(jsonResponse1['dstOffset']) phennyout = loca + ": " + str(jsonResponse1['time']) + ' UTC' + timezone phenny.say(phennyout) except: phenny.say('Something went wrong')
def apply_array_location_correction(phot_groups_filepath): """ Reads the phot_groups.json files and applies the array location correction to all measurements of ALL sources. Writes the result to disk in work_dir as 'phot_groups_arrayloc.json' """ work_dir = "/".join(phot_groups_filepath.split("/")[:-1]) meta = json.load(open(work_dir + "/metadata.json")) # read in the array location correction values root = os.path.abspath("../bcdphot") if meta["channel"] is "1": fp = os.path.join(root, "cal", "ch1_photcorr_ap_5.fits") elif meta["channel"] is "2": fp = os.path.join(root, "cal", "ch2_photcorr_ap_5.fits") arrloc = pyfits.open(fp)[0].data # read in the photometry JSON files ch = json.load(open(phot_groups_filepath)) # apply correction for key in ch: for obs in ch[key]: x, y = obs[4:6] obs[6:] = [i * arrloc[x, y] for i in obs[6:]] # write to disk out_path = work_dir + "/phot_groups_arrayloc.json" with open(out_path, "w") as w: json.dump(ch, w, indent=4 * " ") print("created file: " + out_path)
def search(query, resultnum=None, lang=None, since_id=None): output = [] q = [] rpp = 100 q.append(urlencode({'q': query})) if not since_id is None: q.append(urlencode({'since_id': since_id})) if not lang is None: q.append(urlencode({'lang': lang})) if not resultnum is None: rpp = resultnum q.append(urlencode({'rpp': rpp})) baseurl = 'http://search.twitter.com/search.json' url = baseurl + '?' + '&'.join(q) print url response = urllib2.urlopen(url) data = simplejson.load(response) output.append(data) while 'next_page' in data: url = baseurl + data['next_page'] print url response = urllib2.urlopen(url) data = simplejson.load(response) output.append(data) return output
def __init__(self, resultfile=None): self.serialized_keys = ['options', 'name', 'tests', 'uname', 'wglinfo', 'glxinfo', 'lspci', 'time_elapsed'] self.name = None self.uname = None self.options = None self.glxinfo = None self.lspci = None self.time_elapsed = None self.tests = {} if resultfile: # Attempt to open the json file normally, if it fails then attempt # to repair it. try: raw_dict = json.load(resultfile) except ValueError: raw_dict = json.load(self.__repairFile(resultfile)) # Check that only expected keys were unserialized. for key in raw_dict: if key not in self.serialized_keys: raise Exception('unexpected key in results file: ', str(key)) self.__dict__.update(raw_dict) # Replace each raw dict in self.tests with a TestResult. for (path, result) in self.tests.items(): self.tests[path] = TestResult(result)
def get_manifest(tarfile_path): """Extract and return manifest in tarball. Given a path to a tarfile, which is itself the product of "docker save", this discovers the manifest for the collection of images, which provides version information. """ manifest = [] configjson = None with contextlib.closing(tarfile.open(tarfile_path)) as archive: for member in archive.getmembers(): # find the "manifest.json" file, which points to metadata json file if os.path.basename(member.path) == 'manifest.json': initial_manifest = json.load(archive.extractfile(member)) configjson = initial_manifest[0]['Config'] for member in archive.getmembers(): # get manifest from shasum json file, docker ver > 1.10 if configjson and os.path.basename(member.path) == configjson: image_data = json.load(archive.extractfile(member)) manifest.append(image_data) # find the "json" files, which contain all image metadata # legacy code for docker ver < 1.10 elif not configjson and os.path.basename(member.path) == 'json': image_data = json.load(archive.extractfile(member)) manifest.append(image_data) return manifest
def showkeys(d,root=''): import simplejson import urllib if d == '-': d = simplejson.load(sys.stdin) elif isinstance(d,str): d = simplejson.load(urllib.urlopen(d)) for k,v in d.items(): if isinstance(v,str): print "%s[%s]=%r" % (root,k,v) if isinstance(v,int): print "%s[%s]=%r" % (root,k,v) if isinstance(v,float): print "%s[%s]=%r" % (root,k,v) if isinstance(v,dict): showkeys(v,root = root + '['+k+']') if isinstance(v,list): for i,v in enumerate(v): showkeys(v,root = root + '['+k+']' + '['+str(i)+']')
from urllib.request import urlopen import simplejson as json ChannelID = 'UCCJdqnlqrmTUAYvS6i3nAXA' DeveloperKey = "PUT YOUR KEY HERE" maxResults = '50' CommentsList = [] CommentsUrl = "https://www.googleapis.com/youtube/v3/commentThreads?key=" + DeveloperKey + "&channelId=" + ChannelID + "&part=snippet&maxResults=" +maxResults + "&textFormat=plainText" Results = urlopen(CommentsUrl) Results = json.load(Results) for searchResult in Results.get("items", []): if 'topLevelComment' in searchResult['snippet']: comment = searchResult['snippet']["topLevelComment"] author = comment["snippet"]["authorDisplayName"] text = comment["snippet"]["textDisplay"] print (author, text) CommentsList.append(text) print(CommentsList)
#!/usr/bin/env python # Copyright (c) 2015 SpinPunch. All rights reserved. # Use of this source code is governed by an MIT-style license that can be # found in the LICENSE file. import simplejson as json gamedata = json.load(open('gamedata.json')) freetime = gamedata['store']['free_speedup_time'] def alter_time(t): newt = int(t / 2) # do not create times that are close to the free production time limit # (which often causes order confusion), instead push it under if newt > (freetime - 15) and newt < (freetime + 15): newt = (freetime - 15) return newt if __name__ == '__main__': ret = {} for name, unit in gamedata['units'].iteritems(): old_times = unit['build_time'] if type(old_times) == list: new_times = map(alter_time, old_times) else: new_times = alter_time(old_times)
# data_paths = ['.'] # pass # if "input" in cfg: # cfg["input"] = [l.strip() for l in cfg["input"].split(" ,")] # spit to list # else: # raise ValueError("No input director or file provided to validate.") # files_to_validate = set() incident_counter = 0 for src in cfg["input"]: if os.path.isfile(src): logging.debug("Now validating {0}.".format(src)) # errors in json try: incident = simplejson.load(open(src)) except simplejson.scanner.JSONDecodeError: logging.warning("ERROR: %s did not parse properly. Skipping" % src) # replacing vakudate() with iterating errors - 171206 - GDB for e in validator.iter_errors(incident): offendingPath = '.'.join(str(x) for x in e.path) logging.warning("ERROR in %s. %s %s" % (src, offendingPath, e.message)) # capture errors from main # try: # # validator.validate(incident) # replacing with iterating errors - 171206 GDB # main(incident) # except ValidationError as e: # offendingPath = '.'.join(str(x) for x in e.path) # logging.warning("ERROR in %s. %s %s" % (src, offendingPath, e.message))
import os import sys from bottle import run, Bottle, static_file, request from server.params import params, jsonabort from server.staticroute import staticroutestack import toolz as t import simplejson as json db = json.load(open('data/db.json')) # all the fields present in a production skeleton = { # constant fields (don't get updated) 'title': '', 'author': '', 'characters': {}, 'script': [], # modifiable fields - can change these fields. 'blocking': {}, 'cues': {}, 'line_notes': {}, 'director_notes': {}, } app = Bottle() @app.get('/') def index(): return static_file('index.html', 'client')
def sense_tasks(request, senseId): user = request.user if not user or user not in ADMINS: raise Forbidden('not an admin') sense = get_database().senses.find_one({'_id': senseId}) (old_tasks, new_tasks) = query_tasks(request.corpus, sense['lemma'], sense['_id'], sense.get('pos', 'N')) if request.method == 'GET': # retrieve task statistics info = { 'num_existing': len(old_tasks), 'num_remaining': len(new_tasks) } return Response(json.dumps(info), mimetype="text/javascript") elif request.method == 'POST': stuff = json.load(request.stream) print >> sys.stderr, stuff if stuff['method'] == 'existing': for task in old_tasks: task.set_annotators( sorted( set(task._doc.get('annotators', [])).union(stuff['annotators']))) info = { 'num_existing': len(old_tasks), 'num_remaining': len(new_tasks) } return Response(json.dumps(info), mimetype="text/javascript") elif stuff['method'] == 'remaining': for task in new_tasks: create_new_task(task) task.set_annotators( sorted( set(task._doc.get('annotators', [])).union(stuff['annotators']))) task.save() info = { 'num_existing': len(old_tasks) + len(new_tasks), 'num_remaining': 0 } return Response(json.dumps(info), mimetype="text/javascript") elif stuff['method'] == 'wsdgold': spans = set() task0 = old_tasks[0] for task in old_tasks: if 'wsdgold' not in task._doc.get('annotators', []): spans.update(adjudication_spans(task)) task.set_annotators( sorted( set(task._doc.get('annotators', [])).union(['wsdgold']))) task.set_status('wsdgold', 'ready') task.save() print >> sys.stderr, spans if spans: task_adj = create_adjudication_task(task0, spans) task_adj.save() info = { 'num_existing': len(old_tasks), 'num_remaining': len(spans) } return Response(json.dumps(info), mimetype="text/javascript") print >> sys.stderr, 'huh?'
def __init__(self, serverPushCb, queue=None, path=None, filter=True, bufferDelay=1, retryDelay=5, blackList=None): """ @serverPushCb: callback to be used. It receives 'self' as parameter. It should call self.queueNextServerPush() when it's done to queue the next push. It is guaranteed that the queue is not empty when this function is called. @queue: a item queue that implements IQueue. @path: path to save config. @filter: when True (default), removes all "", None, False, [] or {} entries. @bufferDelay: amount of time events are queued before sending, to reduce the number of push requests rate. This is the delay between the end of a request to initializing a new one. @retryDelay: amount of time between retries when no items were pushed on last serverPushCb call. @blackList: events that shouldn't be sent. """ StatusReceiverMultiService.__init__(self) # Parameters. self.watched = [] self.queue = queue if self.queue is None: self.queue = MemoryQueue() self.queue = IndexedQueue(self.queue) self.path = path self.filter = filter self.bufferDelay = bufferDelay self.retryDelay = retryDelay if not callable(serverPushCb): raise NotImplementedError('Please pass serverPushCb parameter.') def hookPushCb(): # Update the index so we know if the next push succeed or not, don't # update the value when the queue is empty. if not self.queue.nbItems(): return defer.succeed(None) self.lastIndex = self.queue.getIndex() return defer.maybeDeferred(serverPushCb, self) self.serverPushCb = hookPushCb self.blackList = blackList # Other defaults. # IDelayedCall object that represents the next queued push. self.task = None self.stopped = False self.lastIndex = -1 self.state = { 'started': str(datetime.datetime.utcnow()), 'next_id': 1, 'last_id_pushed': 0 } # Try to load back the state. if self.path and os.path.isdir(self.path): state_path = os.path.join(self.path, 'state') if os.path.isfile(state_path): with open(state_path, 'r') as f: self.state.update(json.load(f)) if self.queue.nbItems(): # Last shutdown was not clean, don't wait to send events. self.queueNextServerPush()
def taskcluster_upload(self): auth = os.path.join(os.getcwd(), self.config['taskcluster_credentials_file']) credentials = {} execfile(auth, credentials) client_id = credentials.get('taskcluster_clientId') access_token = credentials.get('taskcluster_accessToken') if not client_id or not access_token: self.warning( 'Skipping S3 file upload: No taskcluster credentials.') return self.activate_virtualenv() dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") upload_env = self.query_upload_env() cwd = dirs['abs_locales_dir'] branch = self.config['branch'] revision = self.query_revision() repo = self.query_l10n_repo() pushinfo = self.vcs_query_pushinfo(repo, revision, vcs='hgtool') pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime(pushinfo.pushdate)) routes_json = os.path.join(self.query_abs_dirs()['abs_mozilla_dir'], 'testing/taskcluster/routes.json') with open(routes_json) as routes_file: contents = json.load(routes_file) templates = contents['l10n'] for locale in locales: output = self.get_output_from_command_m( "%s echo-variable-UPLOAD_FILES AB_CD=%s" % (make, locale), cwd=cwd, env=upload_env, ) files = shlex.split(output) abs_files = [os.path.abspath(os.path.join(cwd, f)) for f in files] routes = [] fmt = { 'index': self.config.get('taskcluster_index', 'index.garbage.staging'), 'project': branch, 'head_rev': revision, 'pushdate': pushdate, 'year': pushdate[0:4], 'month': pushdate[4:6], 'day': pushdate[6:8], 'build_product': self.config['stage_product'], 'build_name': self.query_build_name(), 'build_type': self.query_build_type(), 'locale': locale, } for template in templates: routes.append(template.format(**fmt)) self.info('Using routes: %s' % routes) tc = Taskcluster( branch, pushinfo.pushdate, # Use pushdate as the rank client_id, access_token, self.log_obj, ) task = tc.create_task(routes) tc.claim_task(task) for upload_file in abs_files: tc.create_artifact(task, upload_file) tc.report_completed(task)
#!/usr/bin/env python3 #-*- coding:utf-8 -*- #********************************************************************************************************* #* __ __ __ ______ __ __ _______ _______ * #* | |--.| |.---.-..----.| |--.| __ \.---.-..-----.| |_| |--..-----..----. | | __| * #* | _ || || _ || __|| < | __/| _ || || _| || -__|| _| | - |__ | * #* |_____||__||___._||____||__|__||___| |___._||__|__||____|__|__||_____||__| |_______|_______| * #* http://www.blackpantheros.eu | http://www.blackpanther.hu - kbarcza[]blackpanther.hu * Charles Barcza * #*************************************************************************************(c)2002-2018******** # This is very useful tool if you lost exactly passwd for your Encfs. # Written by Miklos Horvath - [email protected] # the demo passwd is "#EncFS$Rec@ver!123" import itertools import simplejson with open("COMBS", "r") as fp: combs = itertools.product(*simplejson.load(fp)) for e in combs: pwd = "".join(e) print(pwd) # use for test and stop if generated your demo key, just add XXX to key to ignore this line, or delete XXX for use for test if pwd == "XXX#EncFS$Rec@ver!123": print("-- OK -- Last key is your generated demo key!") break
#判斷該路徑的檔案是否存在或是否有資料 if os.path.isfile(file_path) and os.stat(file_path).st_size != 0: #將json檔的資料寫入txt檔內 #雖然他多寫r+,但實際結果感覺一樣 #old_file = open(file_path,"r+") old_file = open(file_path) #json.loads():從json檔讀資料後,將其從str轉換成dictionary #順帶一提,read會回傳str #data = json.loads(old_file.read()) #json.load():直接從json檔讀資料,因此,可以說他具備read()的功能 data = json.load(old_file) write_into(data) else: #如果json檔不存在,就新增一個,然後幫他寫入資料 old_file = open(file_path,"w+") #資料要寫成這樣,才能存入,但有時還是會出現simplejson.errors.JSONDecodeError: Expecting value: line 1 column 1 (char 0) data = {"ages":20, "sex":"male"} #以json檔的資料格式寫入新創造的json檔 #json.dumps則是單純將資料轉乘json格式並回傳資料 old_file.write(json.dumps(data))
import numpy as np VOCAB_SIZE = 2000 TRAINING_FILES = 100 BATCH_SIZE = 256 EPOCHS = 2000 DIMENSIONS = 50 #after how many training samples to save model SAVE_MODEL = 10000 samples = 0 count = 0 w2v = train_word_vectors(VOCAB_SIZE, BATCH_SIZE, EPOCHS, DIMENSIONS) for i in range(TRAINING_FILES): f = open('Samples/samples_{}'.format(i), 'r') data = simplejson.load(f) x_train = np.asarray(data[0]) y_train = np.asarray(data[1]) #print x_train.shape #print y_train.shape del data[:] count += len(x_train) if(count >= SAVE_MODEL): print 'Saved model after {} samples'.format(count) w2v.save('w2v.h5') count = 0 cbk = EarlyStopping(monitor='val_loss', min_delta=0, patience=3, verbose=0, mode='min') w2v.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, validation_split=0.2, callbacks=[cbk], verbose=1) samples += len(x_train) print "Trained on {} samples".format(samples)
from urllib2 import * from solr import * import urllib2 import logging import simplejson from stemming.porter2 import stem logging.basicConfig(format=' %(message)s', level=logging.INFO) from gensim import corpora, models, similarities #conn = urlopen('http://richieverma.koding.io:8983/solr/partB/select?q=tweet_tags:*&wt=json&rows=13000') conn = urlopen('http://52.34.17.82:8983/solr/ezra/select?q=tweet_tags:*&wt=json&rows=11051') initial=True rsp = simplejson.load(conn) f = open('output_lda.txt', 'w') f1 = open('output_lda_ids.txt','w') for doc in rsp['response']['docs']: last_trailing_word = "[^' 'a-zA-Z#:\/0-9._@,]*" pat = re.compile( last_trailing_word ) f1.write(doc['id'] + "\n") for s in doc['tweet_tags']: tweet = pat.sub('',s) #if initial==True: f.write(tweet+" ") #initial=False #else: #f.write("\n" + tweet) f.write('\n') f.close() f1.close()
def get_data(self, **kwargs): if not self.auth: raise AssertionError("You must authenticate() before making requests.") # establish defaults for all variables league = "" season = "" feed = "" output_format = "" params = {} # iterate over args and assign vars for key, value in kwargs.items(): if str(key) == 'league': league = value elif str(key) == 'season': season = value elif str(key) == 'feed': feed = value elif str(key) == 'format': output_format = value else: params[key] = value # add force=false parameter (helps prevent unnecessary bandwidth use) if not "force" in params: params['force'] = 'false' if self.__verify_feed(feed) == False: raise ValueError("Unknown feed '" + feed + "'. Known values are: " + str(self.valid_feeds)) if self.__verify_format(output_format) == False: raise ValueError("Unsupported format '" + output_format + "'.") url = self.determine_url(league, season, feed, output_format, params) if self.verbose: print("Making API request to '{}'.".format(url)) print(" with headers:") print(self.headers) print(" and params:") print(params) r = requests.get(url, params=params, headers=self.headers) if r.status_code == 200: if self.store_type != None: self.__save_feed(r, league, season, feed, output_format, params) if output_format == "json": data = json.loads(r.content) elif output_format == "xml": data = str(r.content) else: data = r.content.splitlines() elif r.status_code == 304: if self.verbose: print("Data hasn't changed since last call") filename = self.__make_output_filename(league, season, feed, output_format, params) with open(self.store_location + filename) as f: if output_format == "json": data = json.load(f) elif output_format == "xml": data = str(f.readlines()[0]) else: data = f.read().splitlines() else: raise Warning("API call failed with error: {error}".format(error=r.status_code)) return data
def qsub(conf_file): tpl = """ #! /bin/sh #? nom du job affiche #PBS -N @exp #PBS -o stdout #PBS -b stderr #PBS -M @email # maximum execution time #PBS -l walltime=@wall_time # mail parameters #PBS -m abe # number of nodes #PBS -l nodes=@nb_cores:ppn=@ppn #PBS -l pmem=5200mb -l mem=5200mb export LD_LIBRARY_PATH=@ld_lib_path exec @exec """ if os.environ.has_key('LD_LIBRARY_PATH'): ld_lib_path = os.environ['LD_LIBRARY_PATH'] else: ld_lib_path = "''" home = os.environ['HOME'] print 'LD_LIBRARY_PATH=' + ld_lib_path # parse conf conf = simplejson.load(open(conf_file)) exps = conf['exps'] nb_runs = conf['nb_runs'] res_dir = conf['res_dir'] bin_dir = conf['bin_dir'] wall_time = conf['wall_time'] use_mpi = "false" try: use_mpi = conf['use_mpi'] except: use_mpi = "false" try: nb_cores = conf['nb_cores'] except: nb_cores = 1 try: args = conf['args'] except: args = '' email = conf['email'] if (use_mpi == "true"): ppn = '1' mpirun = 'mpirun' else: nb_cores = 1 ppn = '8' mpirun = '' for i in range(0, nb_runs): for e in exps: directory = res_dir + "/" + e + "/exp_" + str(i) try: os.makedirs(directory) except: print "WARNING, dir:" + directory + " not be created" subprocess.call('cp ' + bin_dir + '/' + e + ' ' + directory, shell=True) fname = home + "/tmp/" + e + "_" + str(i) + ".job" f = open(fname, "w") f.write( tpl.replace("@exp", e).replace("@email", email).replace( "@ld_lib_path", ld_lib_path).replace("@wall_time", wall_time).replace( "@dir", directory).replace("@nb_cores", str(nb_cores)).replace( "@ppn", ppn).replace( "@exec", mpirun + ' ' + directory + '/' + e + ' ' + args)) f.close() s = "qsub -d " + directory + " " + fname print "executing:" + s retcode = subprocess.call(s, shell=True, env=None) print "qsub returned:" + str(retcode)
def info(self, dircap, path): path = self.fixwinpath(path, False) meta = json.load( self.connection.get(u'/uri/%s%s' % (dircap, path), {u't': u'json'})) return self._info(path, meta)
def get_exp(conf_file): conf = simplejson.load(open(conf_file)) return conf['exp'].sub('exp/', '')
import simplejson, urllib orig_coord = orig_lat, orig_lng dest_coord = dest_lat, dest_lng url = "http://maps.googleapis.com/maps/api/distancematrix/json?origins={0}&destinations={1}&mode=driving&language=en-EN&sensor=false".format(str(orig_coord),str(dest_coord)) result= simplejson.load(urllib.urlopen(url)) driving_time = result['rows'][0]['elements'][0]['duration']['value']
if res[0][0] > 0: print str(res[0][0]) + " " + pre + " already for user [" + user_id + "]: this script will exit..." sys.exit() if user_id > 0: conn = httplib.HTTPConnection("api.twitter.com") try: url = "/1/" + pre + "/ids.json?user_id=" + user_id if cursor is not None: url += "&cursor=" + cursor print "http://api.twitter.com/" + url conn.request("GET", url) except (Exception): sys.exit(sys.exc_info()) r = conn.getresponse() sql = "INSERT INTO " + t + " (source_id, target_id, retrieved) VALUES (" + str(user_id) + ", %d, NOW()) " if r.status == 200: j = simplejson.load(r) if len(j["ids"]) > 0: for x in j["ids"]: #sel = pgconn.query(sql % x) try: print pgconn.insert(t, {"source_id": user_id, "target_id": x, "retrieved": "NOW()"}) except pg.ProgrammingError: print "already exists: " + str(user_id) + "->" + str(x) continue if len(j["ids"]) >= 5000: print j["next_cursor_str"] else: print r.read()
def launch_exp(conf_file): print '--- launch exp ---' # parse configuration print 'launch, conf = ' + conf_file conf = simplejson.load(open(conf_file)) machines = conf['machines'] nb_runs = conf['nb_runs'] exp = conf['exp'] directory = conf['dir'] debug = conf['debug'] args = "" if 'args' in conf: args = conf['args'] print 'exp = ' + exp print 'dir = ' + directory print 'nb_runs = ' + str(nb_runs) print 'debug = ' + str(debug) print 'machines =' + str(machines) print 'args =' + str(args) # copy binaries (debug and opt) & json file exe = get_exe(conf_file) try: os.makedirs(directory + '/bin') os.system('cp ' + 'build/default/' + exp + '/' + exe + ' ' + directory + '/bin/' + exe + '_opt') os.system('cp ' + 'build/debug/' + exp + '/' + exe + ' ' + directory + '/bin/' + exe + '_debug') print conf print directory os.system('cp ' + conf_file + ' ' + directory) # create directories for i in range(0, nb_runs * len(machines)): os.makedirs(directory + '/exp_' + str(i)) except: print '/!\ files exist, I cannot replace them' return print 'dirs created' # make a svn diff status, version = commands.getstatusoutput('svnversion') if version[len(version) - 1] == 'M': version = version[0:len(version) - 1] os.system('svn diff >' + directory + '/' + version + '.diff') print 'diff done [version=' + version + ']' # run on each machines if debug == 1: exe = exe + '_debug' else: exe = exe + '_opt' if os.environ.has_key('LD_LIBRARY_PATH'): ld_lib_path = os.environ['LD_LIBRARY_PATH'] else: ld_lib_path = "''" k = 0 pids = [] for m in machines.iterkeys(): pid = os.fork() if (pid == 0): #son for i in range(0, machines[m]): if m == 'localhost': s = "export LD_LIBRARY_PATH=" + ld_lib_path + \ " && cd " + os.getcwd() + '/' + directory + '/exp_'+ str(i + k) + \ " && " + os.getcwd() + '/' + directory + '/bin/' + exe + " " + args + \ " 1> stdout 2> stderr" else: s = "ssh -o CheckHostIP=false " + m + \ " 'export LD_LIBRARY_PATH=" + ld_lib_path + \ " && cd " + os.getcwd() + '/' + directory + '/exp_'+ str(i + k) + \ " && " + os.getcwd() + '/' + directory + '/bin/' + exe + " " + args + \ " 1> stdout 2> stderr'" print 'run ' + str(i + k) + ' on ' + m print s ret = subprocess.call(s, shell=True) print "ret = " + str(ret) exit(0) pids += [pid] k += machines[m] print "waitpid..." for i in pids: os.waitpid(i, 0)
def load(self): assert op.exists(self.path), self.path logging.debug('%s: loading info from: %s' % (self.__class__.__name__, self.path)) self.info = json.load(open(self.path)) logging.debug(self.dump())
def loadleveler(conf_file): tpl = """ # @ job_name=<name> # @ output = $(job_name).$(jobid) # @ error = $(output) # @ job_type = serial # @ class = <class> # @ resources=ConsumableMemory(<memory>) ConsumableCpus(<cpu>) # @ queue export LD_LIBRARY_PATH=<ld_library_path> cd <initial_dir> ./<exec> """ if os.environ.has_key('LD_LIBRARY_PATH'): ld_lib_path = os.environ['LD_LIBRARY_PATH'] else: ld_lib_path = "''" home = os.environ['HOME'] print 'LD_LIBRARY_PATH=' + ld_lib_path # parse conf conf = simplejson.load(open(conf_file)) jobname = conf['jobname'] exps = conf['exps'] nb_runs = conf['nb_runs'] res_dir = conf['res_dir'] bin_dir = conf['bin_dir'] jobclass = conf['class'] try: memory = conf['memory'] except: memory = 3000 try: cpu = conf['cpu'] except: cpu = 1 for i in range(0, nb_runs): for e in exps: directory = res_dir + "/" + e + "/exp_" + str(i) try: os.makedirs(directory) except: print "WARNING, dir:" + directory + " cannot be created" subprocess.call('cp ' + bin_dir + '/' + e + ' ' + directory, shell=True) try: os.makedirs(home + "/tmp") except: pass fname = home + "/tmp/" + e + "_" + str(i) + ".job" f = open(fname, "w") f.write( tpl.replace("<name>", jobname).replace( "<ld_library_path>", ld_lib_path).replace("<class>", jobclass).replace( "<initial_dir>", directory).replace("<memory>", str(memory)).replace( "<cpu>", str(cpu)).replace("<exec>", e)) f.close() s = "llsubmit " + fname print "executing:" + s retcode = subprocess.call(s, shell=True, env=None) print "llsubmit returned:" + str(retcode)
def insert(self, h=None): if h: return json.dumps(self.cluster.insert(int(h))) return json.dumps(self.cluster.insert(*json.load(request.body)))
def newFromJsonFile(jsonfile): with open(jsonfile) as json_file: json_data = json.load(json_file) return LimitRanges.newFromDict(json_data)
def metric_init(params): descriptors = [] logging.debug('[elasticsearch] Received the following parameters') logging.debug(params) host = params.get('host', 'http://localhost:9200/') version = params.get('version', '1.3') m = re.match('(?P<major>\d+)\.(?P<minor>(\d+(\.\d+)*))', version) if m and m.group('major') == '0': url_cluster = '{0}_cluster/nodes/_local/stats?all=true'.format(host) else: url_cluster = '{0}_nodes/_local/stats'.format(host) # First iteration - Grab statistics logging.debug('[elasticsearch] Fetching ' + url_cluster) result = json.load(urllib.urlopen(url_cluster)) metric_group = params.get('metric_group', 'elasticsearch') Desc_Skel = { 'name': 'XXX', 'call_back': partial(getStat, result, url_cluster), 'time_max': 60, 'value_type': 'uint', 'units': 'units', 'slope': 'both', 'format': '%d', 'description': 'XXX', 'groups': metric_group, } indices = params.get('indices', '*').split() for index in indices: url_indices = '{0}{1}/_stats'.format(host, index) logging.debug('[elasticsearch] Fetching ' + url_indices) r_indices = json.load(urllib.urlopen(url_indices)) descriptors += get_indices_descriptors(index, Desc_Skel, r_indices, url_indices) _create_desc = partial(create_desc, Desc_Skel) descriptors.append( _create_desc({ 'name': 'es_heap_committed', 'units': 'Bytes', 'format': '%.0f', 'description': 'Java Heap Committed (Bytes)', 'value_type': 'double' })) descriptors.append( _create_desc({ 'name': 'es_heap_used', 'units': 'Bytes', 'format': '%.0f', 'description': 'Java Heap Used (Bytes)', 'value_type': 'double' })) descriptors.append( _create_desc({ 'name': 'es_non_heap_committed', 'units': 'Bytes', 'format': '%.0f', 'description': 'Java Non Heap Committed (Bytes)', 'value_type': 'double' })) descriptors.append( _create_desc({ 'name': 'es_non_heap_used', 'units': 'Bytes', 'format': '%.0f', 'description': 'Java Non Heap Used (Bytes)', 'value_type': 'double' })) descriptors.append( _create_desc({ 'name': 'es_threads', 'units': 'threads', 'format': '%d', 'description': 'Threads (open)', })) descriptors.append( _create_desc({ 'name': 'es_threads_peak', 'units': 'threads', 'format': '%d', 'description': 'Threads Peak (open)', })) descriptors.append( _create_desc({ 'name': 'es_gc_young_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Java GC Young Time (ms)' })) descriptors.append( _create_desc({ 'name': 'es_gc_old_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Java GC Old Time (ms)' })) descriptors.append( _create_desc({ 'name': 'es_transport_open', 'units': 'sockets', 'format': '%d', 'description': 'Transport Open (sockets)', })) descriptors.append( _create_desc({ 'name': 'es_transport_rx_count', 'units': 'rx', 'format': '%d', 'slope': 'positive', 'description': 'RX Count' })) descriptors.append( _create_desc({ 'name': 'es_transport_rx_size', 'units': 'Bytes', 'format': '%.0f', 'slope': 'positive', 'description': 'RX (Bytes)', 'value_type': 'double', })) descriptors.append( _create_desc({ 'name': 'es_transport_tx_count', 'units': 'tx', 'format': '%d', 'slope': 'positive', 'description': 'TX Count' })) descriptors.append( _create_desc({ 'name': 'es_transport_tx_size', 'units': 'Bytes', 'format': '%.0f', 'slope': 'positive', 'description': 'TX (Bytes)', })) descriptors.append( _create_desc({ 'name': 'es_http_current_open', 'units': 'sockets', 'format': '%d', 'description': 'HTTP Open (sockets)', })) descriptors.append( _create_desc({ 'name': 'es_http_total_open', 'units': 'sockets', 'format': '%d', 'description': 'HTTP Open (sockets)', })) descriptors.append( _create_desc({ 'name': 'es_indices_size', 'units': 'Bytes', 'format': '%.0f', 'description': 'Index Size (Bytes)', 'value_type': 'double', })) descriptors.append( _create_desc({ 'name': 'es_gc_young_count', 'format': '%d', 'slope': 'positive', 'description': 'Java GC Young Count', })) descriptors.append( _create_desc({ 'name': 'es_gc_old_count', 'format': '%d', 'slope': 'positive', 'description': 'Java GC Old Count', })) descriptors.append( _create_desc({ 'name': 'es_merges_current', 'format': '%d', 'description': 'Merges (current)', })) descriptors.append( _create_desc({ 'name': 'es_merges_current_docs', 'format': '%d', 'description': 'Merges (docs)', })) descriptors.append( _create_desc({ 'name': 'es_merges_total', 'format': '%d', 'slope': 'positive', 'description': 'Merges (total)', })) descriptors.append( _create_desc({ 'name': 'es_merges_total_docs', 'format': '%d', 'slope': 'positive', 'description': 'Merges (total docs)', })) descriptors.append( _create_desc({ 'name': 'es_merges_current_size', 'units': 'Bytes', 'format': '%.0f', 'description': 'Merges size (current)', })) descriptors.append( _create_desc({ 'name': 'es_merges_total_size', 'units': 'Bytes', 'format': '%.0f', 'slope': 'positive', 'description': 'Merges size (total)', 'value_type': 'double', })) descriptors.append( _create_desc({ 'name': 'es_merges_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Merges Time (ms)' })) descriptors.append( _create_desc({ 'name': 'es_refresh_total', 'units': 'refreshes', 'format': '%d', 'slope': 'positive', 'description': 'Total Refresh' })) descriptors.append( _create_desc({ 'name': 'es_refresh_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Total Refresh Time' })) descriptors.append( _create_desc({ 'name': 'es_docs_count', 'units': 'docs', 'format': '%.0f', 'description': 'Number of Documents', 'value_type': 'double' })) descriptors.append( _create_desc({ 'name': 'es_docs_deleted', 'units': 'docs', 'format': '%.0f', 'description': 'Number of Documents Deleted', 'value_type': 'double' })) descriptors.append( _create_desc({ 'name': 'es_open_file_descriptors', 'units': 'files', 'format': '%d', 'description': 'Open File Descriptors', })) descriptors.append( _create_desc({ 'name': 'es_cache_fielddata_evictions', 'units': 'units', 'format': '%d', 'slope': 'positive', 'description': 'Field Data Cache Evictions', })) descriptors.append( _create_desc({ 'name': 'es_cache_fielddata_size', 'units': 'Bytes', 'format': '%.0f', 'description': 'Field Data Cache Size', 'value_type': 'double', })) descriptors.append( _create_desc({ 'name': 'es_cache_filter_evictions', 'format': '%d', 'slope': 'positive', 'description': 'Filter Cache Evictions', })) descriptors.append( _create_desc({ 'name': 'es_cache_filter_size', 'units': 'Bytes', 'format': '%.0f', 'description': 'Filter Cache Size', 'value_type': 'double' })) descriptors.append( _create_desc({ 'name': 'es_query_current', 'units': 'Queries', 'format': '%d', 'description': 'Current Queries', })) descriptors.append( _create_desc({ 'name': 'es_query_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Total Query Time' })) descriptors.append( _create_desc({ 'name': 'es_fetch_current', 'units': 'fetches', 'format': '%d', 'description': 'Current Fetches', })) descriptors.append( _create_desc({ 'name': 'es_fetch_total', 'units': 'fetches', 'format': '%d', 'slope': 'positive', 'description': 'Total Fetches' })) descriptors.append( _create_desc({ 'name': 'es_fetch_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Total Fetch Time' })) descriptors.append( _create_desc({ 'name': 'es_flush_total', 'units': 'flushes', 'format': '%d', 'description': 'Total Flushes', })) descriptors.append( _create_desc({ 'name': 'es_flush_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Total Flush Time' })) descriptors.append( _create_desc({ 'name': 'es_get_exists_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Exists Time' })) descriptors.append( _create_desc({ 'name': 'es_get_exists_total', 'units': 'total', 'format': '%d', 'description': 'Exists Total', })) descriptors.append( _create_desc({ 'name': 'es_get_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Get Time' })) descriptors.append( _create_desc({ 'name': 'es_get_total', 'units': 'total', 'format': '%d', 'description': 'Get Total', })) descriptors.append( _create_desc({ 'name': 'es_get_missing_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Missing Time' })) descriptors.append( _create_desc({ 'name': 'es_get_missing_total', 'units': 'total', 'format': '%d', 'description': 'Missing Total', })) descriptors.append( _create_desc({ 'name': 'es_indexing_delete_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Delete Time' })) descriptors.append( _create_desc({ 'name': 'es_indexing_delete_total', 'units': 'docs', 'format': '%d', 'slope': 'positive', 'description': 'Delete Total' })) descriptors.append( _create_desc({ 'name': 'es_indexing_index_time', 'units': 'ms', 'format': '%d', 'slope': 'positive', 'description': 'Indexing Time' })) descriptors.append( _create_desc({ 'name': 'es_indexing_index_total', 'units': 'docs', 'format': '%d', 'slope': 'positive', 'description': 'Indexing Documents Total' })) descriptors.append( _create_desc({ 'name': 'es_query_total', 'units': 'Queries', 'format': '%d', 'slope': 'positive', 'description': 'Total Queries' })) return descriptors
def remove(self, h=None): if h: return json.dumps(self.cluster.remove(int(h))) return json.dumps(self.cluster.remove(*json.load(request.body)))
def setUp(self): with open("sensor_data_network_config.json", "rb") as jsonFile: self.templateNetworkConfig = json.load(jsonFile)
def all(self, query=None): if query: return json.dumps(self.cluster.find_all(int(query))) return json.dumps(self.cluster.find_all(*json.load(request.body)))
def maybe_get(x): return x if __name__ == "__main__": random.seed(435) np.random.seed(2342) np.set_printoptions(linewidth=100) data_dir = os.path.join("../data", "stanfordmovie") with open( os.path.join(data_dir, "stanfordmovie.train.sentences.clean.projected.json") ) as data_file: data = json.load(data_file) random.shuffle(data) X, Y = map(list, zip(*data)) Y = [[":)", ":("].index(y) for y in Y] with open( os.path.join( data_dir, "stanfordmovie.train.sentences.clean.dictionary.encoding.json") ) as encoding_file: encoding = json.load(encoding_file) print len(encoding) n_validation = 500 batch_size = 25
def update_result(result, url): logging.debug('[elasticsearch] Fetching ' + url) result = json.load(urllib.urlopen(url)) return result
# Standard # ############ import os import shutil from pathlib import Path ############### # Third Party # ############### import pytest import simplejson as sjson from cookiecutter import main CCDSA_ROOT = os.path.abspath(os.path.join(__file__, os.pardir, os.pardir)) TEMPLATE_DIR = Path(CCDSA_ROOT) / "{{ cookiecutter.folder_name }}" CC_JSON = sjson.load(open(str(Path(CCDSA_ROOT) / "cookiecutter.json"))) @pytest.fixture(scope='function') def default_baked_project(tmpdir): out_dir = str(tmpdir.mkdir('data-project')) main.cookiecutter(CCDSA_ROOT, no_input=True, extra_context={}, output_dir=out_dir) # default project name is project_name yield os.path.join(out_dir, 'project_name') # cleanup after
c.execute("CREATE TABLE IF NOT EXISTS users(id integer PRIMARY KEY, username text, phone text, website text, regdate text)") # AUTOINCREMENT : 자동 증가, PRIMARY KEY : 기본 키. 중복이 될 수 없는 값 # 데이터 삽입 c.execute('INSERT INTO users VALUES(1, "kim", "010-000-0000", "kim.co.kr", ?)', (nowDateTime,)) # 매핑은 튜플로 한다. userList = ( (2, "kim", "010-000-0000", "kim.co.kr", nowDateTime), (3, "park", "010-000-0000", "park.co.kr", nowDateTime), (4, "lee", "010-000-0000", "lee.co.kr", nowDateTime), ) # c.executemany("INSERT INTO users(id, username, phone, website, regdate) VALUES(?,?,?,?,?)", userList) with open('data/users.json', 'r') as infile : r = json.load(infile) userData = [] for user in r : tup = (user['id'], user['name'], user['phone'], user['website'], nowDateTime) # print(tup) userData.append(tup) c.execute("INSERT INTO users VALUES(?,?,?,?,?)", tup) # c.executemany("INSERT INTO users(id, username, phone, website, regdate) VALUES(?,?,?,?,?)", userData) # userData리스트를 알아서 튜플로 형변환 해 준다. # 데이터 삭제 # print('users db delete', conn.execute('delete from users').rowcount, "rows") conn.commit()