def render_GET(self, request): # No authz check, we allow everyone to fetch service list host = get_headers(request, 'host') if not host: log.msg('Client did not send proper host header.', system='sgas.TopResource') return # FIXME this returns 500... # stuff needed for being cooperative with a reverse proxy # note: once loggers get updated to understands path referrel # (currently they only understand complete URLs) these hacks # can be removed - this will probably be in the beginning of 2011 :-) if has_headers(request, 'x-forwarded-port'): host += ':' + get_headers(request, 'x-forwarded-port') is_secure = request.isSecure() if get_headers(request, 'x-forwarded-protocol') == 'https': is_secure = True basepath = '/'.join(request.prepath) baseurl = self._createBaseURL(host, is_secure, basepath) #print "BASEURL", baseurl tree = self._createServiceTree(baseurl) ts = XML_HEADER + ET.tostring(tree) + "\n" return ts
def editor(): source = request.args.get('source', default=None) original_file_name = source.split('/')[-1] file_type = request.args.get('format', default="csv") callback_url = request.args.get('callback', default="") error_msg = None warning_msg = None file_name = "" if source is None or source.strip() == '': headers = ["AAA", "BBB", "CCC"] # elif source.startswith('file:/'): # original_file_name = source.split('/')[-1] # headers = util.get_headers(source, file_type=file_type) # file_name = source.split('/')[-1].split('.')[0] + "-" + util.get_random_string(4) + "." + source.split('.')[-1] # if headers == []: # warning_msg = "Can't parse the source file %s" % source else: if DOWNLOAD: r = requests.get(source, allow_redirects=True) if r.status_code == 200: fname = source.split('/')[-1].split( '.')[0] + "-" + util.get_random_string( 4) + "." + source.split('.')[-1] file_name = fname uploaded_file_dir = os.path.join(UPLOAD_DIR, fname) f = open(uploaded_file_dir, 'w') f.write(r.content) f.close() headers = util.get_headers(uploaded_file_dir, file_type=file_type) if headers == []: warning_msg = "Can't parse the source file %s" % source else: error_msg = "the source %s can not be accessed" % source print error_msg headers = [] else: headers = util.get_headers(source, file_type=file_type) if headers == []: warning_msg = "Can't parse the source file %s" % source # if callback_url: # files = {'upload_file': open('file.txt', 'rb')} # values = {'DB': 'photcat', 'OUT': 'csv', 'SHORT': 'short'} # r = requests.post(url, files=files, data=values) # return render_template('msg.html', msg="Your mappings has been sent", msg_title="Result") # else: f = open(os.path.join(DATA_DIR, "labels.txt")) return render_template('editor.html', labels_txt=f.read(), headers=headers, callback=callback_url, file_name=original_file_name, error_msg=error_msg, warning_msg=warning_msg)
def import_data(f, record_id, target_id, id=False): '''imports the data and converts it to X (input) and y (output) data vectors''' rows = read_csv(f) # save column names as headers, save indices of record and target IDs headers = util.get_headers(rows.next()) try: record_col = headers.index(record_id) target_col = headers.index(target_id) except: print 'The specified instance ID was not found as column name. Manually check input file for correct instance ID column.' return False, False, False # save and split records print ' ...(loading)' if not id: records = [row[1:] for row in rows] headers = headers[1:-1] else: records = [row for row in rows] headers = headers[0:-1] print ' ...(converting to matrix)' records = np.matrix(records) X = records[:, 0:-1] # features # output y = records[:, -1] # target y = np.squeeze(np.asarray(y.astype(np.float))) print ' ...(converting data type)' X = X.astype(np.float64, copy=False) y = y.astype(np.float64, copy=False) return X, y, headers
def import_data(f, record_id, target_id): '''imports the data and converts it to X (input) and y (output) data vectors''' rows = read_csv(f) # save column names as headers, save indices of record and target IDs headers = util.get_headers(rows.next()) try: record_col = headers.index(record_id) target_col = headers.index(target_id) except: print 'The specified instance ID was not found as column name. Manually check input file for correct instance ID column.' return False, False, False # save and split records print ' ...(loading)' records = [row[1:] for row in rows] print ' ...(converting to matrix)' records = np.matrix(records) X = records[:,0:-1] # features headers = headers[1:-1] # output y = records[:,-1] # target y=np.squeeze(np.asarray(y.astype(np.int))) print ' ...(converting data type)' X = X.astype(np.float64, copy=False) y = y.astype(np.float64, copy=False) return X, y, headers
def getSubject(request): """ Utility method for extracting the subject name from a twisted.web.http.Request """ # identity forwarded by reverse proxy if request.getClientIP() in LOOPBACK_ADDRESSES and has_headers(request, X_SSL_SUBJECT): return get_headers(request, X_SSL_SUBJECT) # request wasn't secure or no certificate was presented return None
def getSubject(request): """ Utility method for extracting the subject name from a twisted.web.http.Request """ # identity forwarded by reverse proxy if request.getClientIP() in LOOPBACK_ADDRESSES and has_headers( request, X_SSL_SUBJECT): return get_headers(request, X_SSL_SUBJECT) # request wasn't secure or no certificate was presented return None
def render_GET(self, request): # No authz check, we allow everyone to fetch service list host = get_headers(request, 'host') if not host: log.msg('Client did not send proper host header.', system='sgas.TopResource') return # FIXME this returns 500... # stuff needed for being cooperative with a reverse proxy # note: once loggers get updated to understands path referrel # (currently they only understand complete URLs) these hacks # can be removed - this will probably be in the beginning of 2011 :-) if has_headers(request, 'x-forwarded-port'): host += ':' + get_headers(request, 'x-forwarded-port') is_secure = request.isSecure() if get_headers(request, 'x-forwarded-protocol') == 'https': is_secure = True basepath = '/'.join(request.prepath) baseurl = self._createBaseURL(host, is_secure, basepath) #print "BASEURL", baseurl tree = self._createServiceTree(baseurl) ts = XML_HEADER + ET.tostring(tree) + "\n" return ts
def main(couch_params, main_db, avatar_db, master_params, wikifeat_home): # Set up credentials credentials = util.get_credentials(couch_params.adminuser, couch_params.adminpass) global gh, ph gh = util.get_headers(credentials) ph = util.put_headers(credentials) global wf_dir wf_dir = wikifeat_home # Establish a connection to couchdb conn = util.get_connection(couch_params.use_ssl, couch_params.host, couch_params.port) conn.connect() setup_main_db(conn, main_db) setup_user_db(conn) setup_avatar_db(conn, avatar_db) if master_params.skip_master is False: create_master_user(conn, master_params)
def main(couch_params, main_db, avatar_db, master_params, wikifeat_home): # Set up credentials credentials = util.get_credentials(couch_params.adminuser, couch_params.adminpass) global gh, ph gh = util.get_headers(credentials) ph = util.put_headers(credentials) global wf_dir wf_dir = wikifeat_home # Establish a connection to couchdb conn = util.get_connection(couch_params.use_ssl, couch_params.host, couch_params.port) conn.connect() setup_main_db(conn, main_db) setup_user_db(conn) setup_avatar_db(conn, avatar_db) if master_params.skip_master is False: create_master_user(conn, master_params)
def getHostname(request): """ Utility method for getting hostname of client. """ if request.getClientIP() in LOOPBACK_ADDRESSES and has_headers(request, X_FORWARDED_FOR): # nginx typically returns ip addresses addr = get_headers(request, X_FORWARDED_FOR) if isIPAddress(addr): # we really shouldn't do such blocking calls in twisted, # but the twisted dns interface is rather terrible and # odd things happen when using it # Set timeout to 1 second to limit the possible damage try: socket.setdefaulttimeout(1) info = socket.gethostbyaddr(addr) return info[0] except socket.error, msg: log.msg("Error performing reverse lookup: %s" % msg) return addr else: addr
def getHostname(request): """ Utility method for getting hostname of client. """ if request.getClientIP() in LOOPBACK_ADDRESSES and has_headers( request, X_FORWARDED_FOR): # nginx typically returns ip addresses addr = get_headers(request, X_FORWARDED_FOR) if isIPAddress(addr): # we really shouldn't do such blocking calls in twisted, # but the twisted dns interface is rather terrible and # odd things happen when using it # Set timeout to 1 second to limit the possible damage try: socket.setdefaulttimeout(1) info = socket.gethostbyaddr(addr) return info[0] except socket.error, msg: log.msg("Error performing reverse lookup: %s" % msg) return addr else: addr
def insert_artist_genres(): conn, cursor = util.connect2RDS() headers = util.get_headers(client_id, client_secret) cursor.execute("SELECT id FROM artists") artists = [] for (id, ) in cursor.fetchall(): artists.append(id) artist_batch = [artists[i:i + 50] for i in range(0, len(artists), 50)] artist_genres = [] for i in artist_batch: ids = ','.join(i) URL = "https://api.spotify.com/v1/artists/?ids={}".format(ids) r = requests.get(URL, headers=headers) raw = json.loads(r.text) for artist in raw['artists']: for genre in artist['genres']: artist_genres.append({ 'artist_id': artist['id'], 'genre': genre }) for data in artist_genres: util.insert_row(cursor, data, 'artist_genres') conn.commit() cursor.close() sys.exit(0)
def insert_artists(): conn, cursor = util.connect2RDS() headers = util.get_headers() artists = [] with open('artist_list.csv') as f: raw = csv.reader(f) for row in raw: artists.append(row[0]) for a in artists: params = {"q": a, "type": "artist", "limit": "1"} r = requests.get("https://api.spotify.com/v1/search", params=params, headers=headers) raw = json.loads(r.text) artist = {} try: artist_raw = raw['artists']['items'][0] if artist_raw['name'] == params['q']: artist.update({ 'id': artist_raw['id'], 'name': artist_raw['name'], 'followers': artist_raw['followers']['total'], 'popularity': artist_raw['popularity'], 'url': artist_raw['external_urls']['spotify'], 'image_url': artist_raw['images'][0]['url'] }) util.insert_row(cursor, artist, 'artists') except: logging.error('something worng') continue conn.commit() sys.exit(0)
def editor(): if 'format' in request.form: file_type = request.form['format'] else: file_type = 'csv' if 'callback' in request.form: callback_url = request.form['callback'] else: callback_url = "" ontologies = request.form.getlist('ontologies') if len(ontologies) == 0: return render_template('msg.html', msg="You should select at least one ontology", msg_title="Error") print("number of ontologies: " + str(len(ontologies))) print(ontologies) print(request.form) error_msg = None warning_msg = None uploaded = False if 'source' not in request.form or request.form['source'].strip() == "": if 'sourcefile' in request.files: sourcefile = request.files['sourcefile'] if sourcefile.filename != "": original_file_name = sourcefile.filename filename = secure_filename(sourcefile.filename) fname = util.get_random_string(4) + "-" + filename uploaded_file_dir = os.path.join(UPLOAD_DIR, fname) if not os.path.exists(UPLOAD_DIR): os.mkdir(UPLOAD_DIR) sourcefile.save(uploaded_file_dir) uploaded = True else: print("blank source file") else: print('not sourcefile') if not uploaded: return render_template('msg.html', msg="Expecting an input file", msg_title="Error") else: source = request.form['source'] original_file_name = source.split('/')[-1] filename = secure_filename(original_file_name) r = requests.get(source, allow_redirects=True) if r.status_code == 200: fname = util.get_random_string(4) + "-" + filename uploaded_file_dir = os.path.join(UPLOAD_DIR, fname) f = open(uploaded_file_dir, 'w') f.write(r.content) f.close() else: error_msg = "the source %s can not be accessed" % source print(error_msg) return render_template('msg.html', msg=error_msg, msg_title="Error") headers = util.get_headers(uploaded_file_dir, file_type=file_type) if headers == []: error_msg = "Can't parse the source file " return render_template('msg.html', msg=error_msg, msg_title="Error") labels = util.get_classes_as_txt(ontologies, data_dir=DATA_DIR) # f = open(os.path.join(DATA_DIR, "labels.txt")) return render_template('editor.html', labels_txt=labels, ontologies_txt=",".join(ontologies), headers=headers, callback=callback_url, file_name=fname, error_msg=error_msg, warning_msg=warning_msg)
def headers(conf_path): return get_headers(conf_path) + ["issued eai", "currency seat ordinal"]
def toS3(): conn, cursor = util.connect2RDS() headers = util.get_headers() cursor.execute("SELECT id FROM artists") top_track_keys = { 'id':'id', 'name':'name', 'popularity':'popularity', 'external_urls':'external_urls.spotify' } top_tracks = [] for(id, ) in cursor.fetchall(): URL = "https://api.spotify.com/v1/artists/{}/top-tracks".format(id) params = { 'country':'US' } r = requests.get(URL, params = params, headers = headers) raw = json.loads(r.text) for i in raw['tracks']: top_track = {} for k, v in top_track_keys.items(): top_track.update({k:jsonpath.jsonpath(i,v)[0]}) top_track.update({'artist_id':id}) top_tracks.append(top_track) top_tracks = pd.DataFrame(top_tracks) top_tracks.to_parquet('top-tracks.parquet',engine='pyarrow',compression='snappy') track_ids = [top_tracks.loc[i,'id'] for i in top_tracks.index] dt = datetime.datetime.utcnow().strftime("%Y-%m-%d") s3 = boto3.resource('s3') object = s3.Object('kihong-spotify-lambda','top-tracks/dt={}/top-tracks.parquet'.format(dt)) data = open('top-tracks.parquet','rb') object.put(Body=data) ######################################################################################################## # # # ######################################################################################################## tracks_batch = [track_ids[i:i+100] for i in range(0,len(track_ids),100)] audio_features = [] for i in tracks_batch: ids = ','.join(i) URL = "https://api.spotify.com/v1/audio-features/?ids={}".format(ids) r = requests.get(URL,headers=headers) raw = json.loads(r.text) audio_features.extend(raw['audio_features']) audio_features = pd.DataFrame(audio_features) audio_features.to_parquet('audio-features.parquet',engine='pyarrow',compression='snappy') object = s3.Object('kihong-spotify-lambda', 'audio-features/dt={}/audio-features.parquet'.format(dt)) data = open('audio-features.parquet', 'rb') object.put(Body=data)
def editor(): if 'format' in request.form: file_type = request.form['format'] else: file_type = 'csv' if 'callback' in request.form: callback_url = request.form['callback'] else: callback_url = "" kg = None if 'kg' in request.form: if request.form['kg'].strip() != "": kg = request.form['kg'].strip() ontologies = request.form.getlist('ontologies') if len(ontologies) == 0: return render_template( 'msg.html', msg="You should select at least one ontology", msg_title="Error") logger.debug("number of ontologies: " + str(len(ontologies))) logger.debug(str(ontologies)) logger.debug(str(request.form)) error_msg = None warning_msg = None uploaded = False if 'source' not in request.form or request.form['source'].strip( ) == "": if 'sourcefile' in request.files: sourcefile = request.files['sourcefile'] if sourcefile.filename != "": original_file_name = sourcefile.filename filename = secure_filename(sourcefile.filename) fname = util.get_random_string(4) + "-" + filename uploaded_file_dir = os.path.join(UPLOAD_DIR, fname) if not os.path.exists(UPLOAD_DIR): os.makedirs(UPLOAD_DIR) sourcefile.save(uploaded_file_dir) uploaded = True else: logger.debug("blank source file") else: logger.debug('not sourcefile') if not uploaded: return render_template('msg.html', msg="Expecting an input file", msg_title="Error") else: source = request.form['source'] original_file_name = source.split('/')[-1] filename = secure_filename(original_file_name) r = requests.get(source, allow_redirects=True) if r.status_code == 200: fname = util.get_random_string(4) + "-" + filename uploaded_file_dir = os.path.join(UPLOAD_DIR, fname) if not os.path.exists(UPLOAD_DIR): os.makedirs(UPLOAD_DIR) f = open(uploaded_file_dir, 'w', encoding='utf-8') f.write(r.text) f.close() else: error_msg = "the source %s can not be accessed" % source logger.debug(error_msg) return render_template('msg.html', msg=error_msg, msg_title="Error") headers = util.get_headers(uploaded_file_dir, file_type=file_type) if headers == []: error_msg = "Can't parse the source file " return render_template('msg.html', msg=error_msg, msg_title="Error") logger.debug("headers: ") logger.debug(str(headers)) labels = "" for o in ontologies: o_labels = None try: o_labels = util.get_classes_as_txt([o], data_dir=DATA_DIR) except: o_labels = util.get_classes_as_txt([o], data_dir=ONT_DIR) if o_labels: labels += o_labels logger.debug("labels: ") logger.debug(str(labels)) return render_template('editor.html', labels_txt=labels, ontologies_txt=",".join(ontologies), headers=headers, kg=kg, callback=callback_url, file_name=fname, error_msg=error_msg, warning_msg=warning_msg)