def overpass_nsrid(nsrid='*',
bbox_scandinavia = '[bbox=3.33984375,57.468589192089325,38.408203125,81.1203884020757]'):
# bbox_scandinavia: limit request size, should contains all of Norway.
filename = 'overpass_api_cache_%s_%s.xml' % (nsrid, bbox_scandinavia)
filename = filename.replace(',', '')
filename = filename.replace('*', '-')
filename = filename.replace('[', '')
filename = filename.replace(']', '')
filename = filename.replace('=', '')
logger.debug('cached overpass filename "%s"', filename)
cached, outdated = file_util.cached_file(filename, old_age_days=1)
if cached is not None and not(outdated):
return cached
r = request_session.get('http://www.overpass-api.de/api/xapi_meta?*[no-barnehage:nsrid=%s]%s' % (nsrid, bbox_scandinavia))
ret = r.content
if r.status_code == 200:
file_util.write_file(filename, ret)
return ret
else:
logger.error('Invalid status code %s', r.status_code)
return None
def generate_mix_features(source_dir_path,
target_dir_path,
params_list,
group_size=300,
bin_number=10):
# print "script: extract_feature.py, lineNumber:", sys._getframe().f_lineno, ", func:", sys._getframe().f_code.co_name
for root, _, files in os.walk(source_dir_path):
for file in files:
data = file_util.read_file(source_dir_path + "\\" + file)
# #---------------test function-------------------------
# denoised_data = data_process.denoise(data, settings.THRESHOLDS)
# file_util.write_file('denoised'+'\\'+file, denoised_data)
fp = extract_feature.get_feature_from_matrix(
data, group_size, bin_number)
# fp = extract_feature.get_feature_from_matrix(denoised_data, group_size, bin_number)
fp_dict = dict()
for i, row in enumerate(fp):
row = data_process.normalize(row) #归一化
fp_dict[i] = row
mix_fp_temp = list_util.merge(fp_dict[0], fp_dict[1])
mix_fp = list_util.merge(mix_fp_temp, fp_dict[2])
target_file = os.path.join(target_dir_path,
file[:-4] + "_" + ".txt")
file_util.write_file(target_file, mix_fp)
def download(root_url, directory, filename):
url = os.path.join(root_url, directory, filename) # should probably use some html join, but I know the input...
filename = os.path.join(directory, filename)
response = urllib2.urlopen(url)
content = response.read()
file_util.write_file(filename, content)
return filename
def copy_template_file(out_dir, out_file, opts, in_file=None, rdir=None):
in_path, out_path = create_in_out_pair(out_dir, out_file, in_file, rdir)
if not out_file == psettings.JQUERY:
fname = in_file or out_file
rendered = templater.render_base_templates(in_path, fname, opts)
file_util.write_file(out_path, rendered)
else:
file_util.copy_file_safe(in_path, out_path)
def barnehagefakta_get_json(nbr_id, old_age_days=5, cache_dir='data', keep_history=True):
"""Returns json string for the given nbr_id, caches result to file in directory cache_dir.
If the cached result is older than old_age_days a new version is fetched.
By default (if keep_history is True) changes in the response will detected
and archived for further processing.
In other words:
(1) The first time this is called, barnehagefakta.no/api/barnehage/{nbr_id} is visited,
the response is stored in cache_dir/barnehagefakta_no_nbrId{nbr_id}.json,
the file may consist of only the string '404' if the request returned 404.
(2a) Calling the function again with the same {nbr_id} within old_age_days,
will simply return the content of the previously stored file
(2b) Calling the function again with the same {nbr_id} after old_age_days has passed,
will visit barnehagefakta again, refreshing and returning the local .json file.
If the responce has changed from last time, the previous result is archived as
cache_dir/barnehagefakta_no_nbrId{nbr_id}-{%Y-%m-%d}-OUTDATED.json
May raise requests.ConnectionError if the connection fails.
"""
filename = os.path.join(cache_dir, 'barnehagefakta_no_nbrId{0}.json'.format(nbr_id))
cached, outdated = file_util.cached_file(filename, old_age_days)
if cached is not None and not(outdated):
return cached
# else, else:
url = 'http://barnehagefakta.no/api/barnehage/{0}'.format(nbr_id)
# try:
r = request_session.get(url)
# except requests.ConnectionError as e:
# logger.error('Could not connect to %s, try again later? %s', url, e)
# return None
logger.info('requested %s, got %s', url, r)
ret = None
if r.status_code == 200:
ret = r.content
elif r.status_code == 404:
ret = '404'
else:
logger.error('Unknown status code %s', r.status_code)
if ret is not None:
if keep_history and cached is not None and not(equal_json_responses(ret, cached)): # avoid overriding previous cache
d = datetime.utcnow()
# note: the date will represent the date we discovered this to be outdated
# which is not all that logical, but we just need a unique filename (assuming old_age_days > 1).
logger.warning('Change in response for id=%s, archiving old result', nbr_id)
file_util.rename_file(filename, d.strftime("-%Y-%m-%d-OUTDATED")) # move old one
#return ret, cached
file_util.write_file(filename, ret) # write
return ret
def create(arg_hash):
# Read in the raw strings from the contents of the files.
input_dir = arg_hash['in_dir']
out_dir = arg_hash['out_dir']
if arg_hash['clean']:
print 'cleaning'
initr.clean_dirs(input_dir, out_dir)
if not arg_hash['test']:
sys.exit(1)
options = initr.init_or_read_opts(input_dir, out_dir,
arg_hash['clean_init'])
raw_contents = []
if os.path.isdir(input_dir):
raw_contents = file_util.read_files(input_dir)
else:
raw_contents = [ file_util.read_file(input_dir) ]
if raw_contents == []:
"Couldn't find any blog file(s) in: %s" % input_dir
sys.exit(0)
# Build the blog parser so we can parse the raw strings
blog_parser = parser.buildParser("yaml", options)
# Parse the YAML files into a blog object
blog = blog_parser.parse(raw_contents)
# Sort the posts by date
# TODO(josh): Should this be internal to the blog?
blog.sort_posts()
# Create the links
blog.create_links()
# Display the AST (for the curious)
# print blog.display_ast()
# Generate the JSON representation
json_out = "var pyrite_data = " + (blog.generate_json().strip()) + ";"
# print json_out
# Create the necessary pyrite directories
json_path = os.path.join(out_dir, psettings.JS_DIR, psettings.DATA_FILE)
file_util.write_file(json_path, json_out)
def create(arg_hash):
# Read in the raw strings from the contents of the files.
input_dir = arg_hash['in_dir']
out_dir = arg_hash['out_dir']
if arg_hash['clean']:
print 'cleaning'
initr.clean_dirs(input_dir, out_dir)
if not arg_hash['test']:
sys.exit(1)
options = initr.init_or_read_opts(input_dir, out_dir,
arg_hash['clean_init'])
raw_contents = []
if os.path.isdir(input_dir):
raw_contents = file_util.read_files(input_dir)
else:
raw_contents = [file_util.read_file(input_dir)]
if raw_contents == []:
"Couldn't find any blog file(s) in: %s" % input_dir
sys.exit(0)
# Build the blog parser so we can parse the raw strings
blog_parser = parser.buildParser("yaml", options)
# Parse the YAML files into a blog object
blog = blog_parser.parse(raw_contents)
# Sort the posts by date
# TODO(josh): Should this be internal to the blog?
blog.sort_posts()
# Create the links
blog.create_links()
# Display the AST (for the curious)
# print blog.display_ast()
# Generate the JSON representation
json_out = "var pyrite_data = " + (blog.generate_json().strip()) + ";"
# print json_out
# Create the necessary pyrite directories
json_path = os.path.join(out_dir, psettings.JS_DIR, psettings.DATA_FILE)
file_util.write_file(json_path, json_out)
def wrapper_cached(self,
callback,
url,
cache_filename,
old_age_days=30,
file_mode='',
**kwargs):
cached, outdated = file_util.cached_file(cache_filename,
old_age_days,
mode='r' + file_mode)
if cached is not None and not (outdated):
#logger.info('returning cached %s %s', cached is not None, not(outdated))
return cached
# # Hmm, getting some half-downloaded files with requests library, lets try urllib instead
# try:
# urlretrieve(url, cache_filename)
# except Exception as e:
# try: os.remove(cache_filename) # ensure we don't have a half finished download
# except: pass
# logger.error('Failure downloading %s, %s', url, e)
# return None
# return file_util.read_file(cache_filename)
try:
r = callback(url, **kwargs) #self.get(url, **kwargs)
except requests.ConnectionError as e:
logger.error('Could not connect to %s, try again later? %s', url,
e)
return None
logger.info('requested %s %s, got %s', url, cache_filename, r)
if r.status_code == 200:
ret = r.content
file_util.write_file(cache_filename, ret, mode='w' + file_mode)
return ret
else:
logger.error('Invalid status code %s', r.status_code)
return None
def update_file(filename):
oldcontents = read_file(filename)
if len(oldcontents) == 0:
msg(filename, "empty")
return
if os.path.splitext(filename)[1] == ".py":
# Format Python files using YAPF.
newcontents = yapf_format(filename, oldcontents)
else:
# Format C/C++/ObjC/Java files using clang-format.
newcontents = clang_format(filename, oldcontents)
if newcontents is None:
raise Exception("Failed to process %s" % filename)
if newcontents != oldcontents:
msg(filename, "fixed")
global updatect
updatect += 1
write_file(filename, newcontents)
else:
msg(filename, "ok")
return
def update_file(filename):
oldcontents = read_file(filename)
if len(oldcontents) == 0:
msg(filename, "empty")
return
if os.path.splitext(filename)[1] == ".py":
# Format Python files using YAPF.
newcontents = yapf_format(filename, oldcontents)
else:
# Format C/C++/ObjC/Java files using clang-format.
newcontents = clang_format(filename, oldcontents)
if newcontents is None:
raise Exception("Failed to process %s" % filename)
if newcontents != oldcontents:
msg(filename, "fixed")
global updatect
updatect += 1
write_file(filename, newcontents)
else:
msg(filename, "ok")
return
def init_indir_and_options(input_dir):
ensure_base_dir_exists(input_dir)
print '------------------'
print 'Pyrite Initializer'
print '------------------'
options = read_default_options()
print 'Let\'s initialize your blog with some initial settigns'
print 'You can change them later in the generated pyrite_options.py'
print ''
print 'What do you want your blog to be called?'
blog_name = raw_input('[default=PyriteBlog]: ')
if blog_name == '': blog_name = 'PyriteBlog'
print ''
print 'Your blog will be called: %s' % blog_name
print '------------------'
blog_name
options = options.replace('MyBlogName', blog_name)
print 'Where do you want your blog-files to be read from?'
input_location = raw_input('[default=%s]: ' % (input_dir))
if input_location == '': input_location = input_dir
options = options.replace('my_in_location', input_location)
print '\nWhere do you want your generated blog-files to be put?'
output_location = raw_input('[default=%s]: ' % (
os.path.join(input_dir, 'pyrite_output')))
if output_location == '':
output_location = os.path.join(input_dir, 'pyrite_output')
options = options.replace('my_out_location', output_location)
print '------------------'
print 'Writing your options to %s' % psettings.OPTIONS_NAME
print '------------------'
file_util.write_file(os.path.join(
input_dir, psettings.OPTIONS_NAME), options)
'INCLUDE',
'LIB',
'PATH',
]
for var in required_vars:
if not var in os.environ.keys():
raise Exception('%s environment variable must be set' % var)
# Windows custom toolchain requirements. See comments in gn_args.py.
gn_args['visual_studio_path'] = os.environ['GYP_MSVS_OVERRIDE_PATH']
gn_args['visual_studio_version'] = os.environ['GYP_MSVS_VERSION']
gn_args['visual_studio_runtime_dirs'] = os.environ['VS_CRT_ROOT']
gn_args['windows_sdk_path'] = os.environ['SDK_ROOT']
configs = GetAllPlatformConfigs(gn_args)
for dir, config in configs.items():
# Create out directories and write the args.gn file.
out_path = os.path.join(src_dir, 'out', dir)
make_dir(out_path, False)
args_gn_path = os.path.join(out_path, 'args.gn')
args_gn_contents = GetConfigFileContents(config)
write_file(args_gn_path, args_gn_contents)
# Generate the Ninja config.
cmd = ['gn', 'gen', os.path.join('out', dir)]
if 'GN_ARGUMENTS' in os.environ.keys():
cmd.extend(os.environ['GN_ARGUMENTS'].split(' '))
RunAction(src_dir, cmd)
if platform == 'windows':
issue_1999.apply(out_path)
'INCLUDE',
'LIB',
'PATH',
]
for var in required_vars:
if not var in os.environ.keys():
raise Exception('%s environment variable must be set' % var)
# Windows custom toolchain requirements. See comments in gn_args.py.
gn_args['visual_studio_path'] = os.environ['GYP_MSVS_OVERRIDE_PATH']
gn_args['visual_studio_version'] = os.environ['GYP_MSVS_VERSION']
gn_args['visual_studio_runtime_dirs'] = os.environ['VS_CRT_ROOT']
gn_args['windows_sdk_path'] = os.environ['SDK_ROOT']
configs = GetAllPlatformConfigs(gn_args)
for dir, config in configs.items():
# Create out directories and write the args.gn file.
out_path = os.path.join(src_dir, 'out', dir)
make_dir(out_path, False)
args_gn_path = os.path.join(out_path, 'args.gn')
args_gn_contents = GetConfigFileContents(config)
write_file(args_gn_path, args_gn_contents)
# Generate the Ninja config.
cmd = ['gn', 'gen', os.path.join('out', dir)]
if 'GN_ARGUMENTS' in os.environ.keys():
cmd.extend(os.environ['GN_ARGUMENTS'].split(' '))
RunAction(src_dir, cmd)
if platform == 'windows':
issue_1999.apply(out_path)
