def overpass_nsrid(nsrid='*', bbox_scandinavia = '[bbox=3.33984375,57.468589192089325,38.408203125,81.1203884020757]'): # bbox_scandinavia: limit request size, should contains all of Norway. filename = 'overpass_api_cache_%s_%s.xml' % (nsrid, bbox_scandinavia) filename = filename.replace(',', '') filename = filename.replace('*', '-') filename = filename.replace('[', '') filename = filename.replace(']', '') filename = filename.replace('=', '') logger.debug('cached overpass filename "%s"', filename) cached, outdated = file_util.cached_file(filename, old_age_days=1) if cached is not None and not(outdated): return cached r = request_session.get('http://www.overpass-api.de/api/xapi_meta?*[no-barnehage:nsrid=%s]%s' % (nsrid, bbox_scandinavia)) ret = r.content if r.status_code == 200: file_util.write_file(filename, ret) return ret else: logger.error('Invalid status code %s', r.status_code) return None
def generate_mix_features(source_dir_path, target_dir_path, params_list, group_size=300, bin_number=10): # print "script: extract_feature.py, lineNumber:", sys._getframe().f_lineno, ", func:", sys._getframe().f_code.co_name for root, _, files in os.walk(source_dir_path): for file in files: data = file_util.read_file(source_dir_path + "\\" + file) # #---------------test function------------------------- # denoised_data = data_process.denoise(data, settings.THRESHOLDS) # file_util.write_file('denoised'+'\\'+file, denoised_data) fp = extract_feature.get_feature_from_matrix( data, group_size, bin_number) # fp = extract_feature.get_feature_from_matrix(denoised_data, group_size, bin_number) fp_dict = dict() for i, row in enumerate(fp): row = data_process.normalize(row) #归一化 fp_dict[i] = row mix_fp_temp = list_util.merge(fp_dict[0], fp_dict[1]) mix_fp = list_util.merge(mix_fp_temp, fp_dict[2]) target_file = os.path.join(target_dir_path, file[:-4] + "_" + ".txt") file_util.write_file(target_file, mix_fp)
def download(root_url, directory, filename): url = os.path.join(root_url, directory, filename) # should probably use some html join, but I know the input... filename = os.path.join(directory, filename) response = urllib2.urlopen(url) content = response.read() file_util.write_file(filename, content) return filename
def copy_template_file(out_dir, out_file, opts, in_file=None, rdir=None): in_path, out_path = create_in_out_pair(out_dir, out_file, in_file, rdir) if not out_file == psettings.JQUERY: fname = in_file or out_file rendered = templater.render_base_templates(in_path, fname, opts) file_util.write_file(out_path, rendered) else: file_util.copy_file_safe(in_path, out_path)
def barnehagefakta_get_json(nbr_id, old_age_days=5, cache_dir='data', keep_history=True): """Returns json string for the given nbr_id, caches result to file in directory cache_dir. If the cached result is older than old_age_days a new version is fetched. By default (if keep_history is True) changes in the response will detected and archived for further processing. In other words: (1) The first time this is called, barnehagefakta.no/api/barnehage/{nbr_id} is visited, the response is stored in cache_dir/barnehagefakta_no_nbrId{nbr_id}.json, the file may consist of only the string '404' if the request returned 404. (2a) Calling the function again with the same {nbr_id} within old_age_days, will simply return the content of the previously stored file (2b) Calling the function again with the same {nbr_id} after old_age_days has passed, will visit barnehagefakta again, refreshing and returning the local .json file. If the responce has changed from last time, the previous result is archived as cache_dir/barnehagefakta_no_nbrId{nbr_id}-{%Y-%m-%d}-OUTDATED.json May raise requests.ConnectionError if the connection fails. """ filename = os.path.join(cache_dir, 'barnehagefakta_no_nbrId{0}.json'.format(nbr_id)) cached, outdated = file_util.cached_file(filename, old_age_days) if cached is not None and not(outdated): return cached # else, else: url = 'http://barnehagefakta.no/api/barnehage/{0}'.format(nbr_id) # try: r = request_session.get(url) # except requests.ConnectionError as e: # logger.error('Could not connect to %s, try again later? %s', url, e) # return None logger.info('requested %s, got %s', url, r) ret = None if r.status_code == 200: ret = r.content elif r.status_code == 404: ret = '404' else: logger.error('Unknown status code %s', r.status_code) if ret is not None: if keep_history and cached is not None and not(equal_json_responses(ret, cached)): # avoid overriding previous cache d = datetime.utcnow() # note: the date will represent the date we discovered this to be outdated # which is not all that logical, but we just need a unique filename (assuming old_age_days > 1). logger.warning('Change in response for id=%s, archiving old result', nbr_id) file_util.rename_file(filename, d.strftime("-%Y-%m-%d-OUTDATED")) # move old one #return ret, cached file_util.write_file(filename, ret) # write return ret
def create(arg_hash): # Read in the raw strings from the contents of the files. input_dir = arg_hash['in_dir'] out_dir = arg_hash['out_dir'] if arg_hash['clean']: print 'cleaning' initr.clean_dirs(input_dir, out_dir) if not arg_hash['test']: sys.exit(1) options = initr.init_or_read_opts(input_dir, out_dir, arg_hash['clean_init']) raw_contents = [] if os.path.isdir(input_dir): raw_contents = file_util.read_files(input_dir) else: raw_contents = [ file_util.read_file(input_dir) ] if raw_contents == []: "Couldn't find any blog file(s) in: %s" % input_dir sys.exit(0) # Build the blog parser so we can parse the raw strings blog_parser = parser.buildParser("yaml", options) # Parse the YAML files into a blog object blog = blog_parser.parse(raw_contents) # Sort the posts by date # TODO(josh): Should this be internal to the blog? blog.sort_posts() # Create the links blog.create_links() # Display the AST (for the curious) # print blog.display_ast() # Generate the JSON representation json_out = "var pyrite_data = " + (blog.generate_json().strip()) + ";" # print json_out # Create the necessary pyrite directories json_path = os.path.join(out_dir, psettings.JS_DIR, psettings.DATA_FILE) file_util.write_file(json_path, json_out)
def create(arg_hash): # Read in the raw strings from the contents of the files. input_dir = arg_hash['in_dir'] out_dir = arg_hash['out_dir'] if arg_hash['clean']: print 'cleaning' initr.clean_dirs(input_dir, out_dir) if not arg_hash['test']: sys.exit(1) options = initr.init_or_read_opts(input_dir, out_dir, arg_hash['clean_init']) raw_contents = [] if os.path.isdir(input_dir): raw_contents = file_util.read_files(input_dir) else: raw_contents = [file_util.read_file(input_dir)] if raw_contents == []: "Couldn't find any blog file(s) in: %s" % input_dir sys.exit(0) # Build the blog parser so we can parse the raw strings blog_parser = parser.buildParser("yaml", options) # Parse the YAML files into a blog object blog = blog_parser.parse(raw_contents) # Sort the posts by date # TODO(josh): Should this be internal to the blog? blog.sort_posts() # Create the links blog.create_links() # Display the AST (for the curious) # print blog.display_ast() # Generate the JSON representation json_out = "var pyrite_data = " + (blog.generate_json().strip()) + ";" # print json_out # Create the necessary pyrite directories json_path = os.path.join(out_dir, psettings.JS_DIR, psettings.DATA_FILE) file_util.write_file(json_path, json_out)
def wrapper_cached(self, callback, url, cache_filename, old_age_days=30, file_mode='', **kwargs): cached, outdated = file_util.cached_file(cache_filename, old_age_days, mode='r' + file_mode) if cached is not None and not (outdated): #logger.info('returning cached %s %s', cached is not None, not(outdated)) return cached # # Hmm, getting some half-downloaded files with requests library, lets try urllib instead # try: # urlretrieve(url, cache_filename) # except Exception as e: # try: os.remove(cache_filename) # ensure we don't have a half finished download # except: pass # logger.error('Failure downloading %s, %s', url, e) # return None # return file_util.read_file(cache_filename) try: r = callback(url, **kwargs) #self.get(url, **kwargs) except requests.ConnectionError as e: logger.error('Could not connect to %s, try again later? %s', url, e) return None logger.info('requested %s %s, got %s', url, cache_filename, r) if r.status_code == 200: ret = r.content file_util.write_file(cache_filename, ret, mode='w' + file_mode) return ret else: logger.error('Invalid status code %s', r.status_code) return None
def update_file(filename): oldcontents = read_file(filename) if len(oldcontents) == 0: msg(filename, "empty") return if os.path.splitext(filename)[1] == ".py": # Format Python files using YAPF. newcontents = yapf_format(filename, oldcontents) else: # Format C/C++/ObjC/Java files using clang-format. newcontents = clang_format(filename, oldcontents) if newcontents is None: raise Exception("Failed to process %s" % filename) if newcontents != oldcontents: msg(filename, "fixed") global updatect updatect += 1 write_file(filename, newcontents) else: msg(filename, "ok") return
def init_indir_and_options(input_dir): ensure_base_dir_exists(input_dir) print '------------------' print 'Pyrite Initializer' print '------------------' options = read_default_options() print 'Let\'s initialize your blog with some initial settigns' print 'You can change them later in the generated pyrite_options.py' print '' print 'What do you want your blog to be called?' blog_name = raw_input('[default=PyriteBlog]: ') if blog_name == '': blog_name = 'PyriteBlog' print '' print 'Your blog will be called: %s' % blog_name print '------------------' blog_name options = options.replace('MyBlogName', blog_name) print 'Where do you want your blog-files to be read from?' input_location = raw_input('[default=%s]: ' % (input_dir)) if input_location == '': input_location = input_dir options = options.replace('my_in_location', input_location) print '\nWhere do you want your generated blog-files to be put?' output_location = raw_input('[default=%s]: ' % ( os.path.join(input_dir, 'pyrite_output'))) if output_location == '': output_location = os.path.join(input_dir, 'pyrite_output') options = options.replace('my_out_location', output_location) print '------------------' print 'Writing your options to %s' % psettings.OPTIONS_NAME print '------------------' file_util.write_file(os.path.join( input_dir, psettings.OPTIONS_NAME), options)
'INCLUDE', 'LIB', 'PATH', ] for var in required_vars: if not var in os.environ.keys(): raise Exception('%s environment variable must be set' % var) # Windows custom toolchain requirements. See comments in gn_args.py. gn_args['visual_studio_path'] = os.environ['GYP_MSVS_OVERRIDE_PATH'] gn_args['visual_studio_version'] = os.environ['GYP_MSVS_VERSION'] gn_args['visual_studio_runtime_dirs'] = os.environ['VS_CRT_ROOT'] gn_args['windows_sdk_path'] = os.environ['SDK_ROOT'] configs = GetAllPlatformConfigs(gn_args) for dir, config in configs.items(): # Create out directories and write the args.gn file. out_path = os.path.join(src_dir, 'out', dir) make_dir(out_path, False) args_gn_path = os.path.join(out_path, 'args.gn') args_gn_contents = GetConfigFileContents(config) write_file(args_gn_path, args_gn_contents) # Generate the Ninja config. cmd = ['gn', 'gen', os.path.join('out', dir)] if 'GN_ARGUMENTS' in os.environ.keys(): cmd.extend(os.environ['GN_ARGUMENTS'].split(' ')) RunAction(src_dir, cmd) if platform == 'windows': issue_1999.apply(out_path)