async def upload_picture(field):
# TODO: should check content for actual picture; should also resize on the client side
extension = field.filename.split('.')[-1].lower()
if extension not in ['jpg', 'jpeg', 'png', 'gif']:
raise web.HTTPBadRequest(
reason=
'Picture file type not allowed, please use jpg, jpeg, png or gif')
base = str(uuid.uuid4())
orig_filename = 'orig-%s.%s' % (base, extension)
filename = '%s.jpg' % base
size = 0
with open('./data/pictures/' + orig_filename, 'wb') as f:
while True:
chunk = await field.read_chunk() # 8192 bytes by default.
if not chunk:
break
size += len(chunk)
f.write(chunk)
if size > 1024 * 1024 * 10: # max 10M
fp.close()
os.unlink('./data/pictures/' + orig_filename)
raise web.HTTPBadRequest(reason='Picture file too large')
#status, stdout, stderr = await run_command('./src/make-thumbnail.sh "./data/pictures/%s" "./data/pictures/%s"' % (orig_filename, filename))
if not thumbnail.make_thumbnail('./data/pictures/' + orig_filename,
'./data/pictures/' + filename, 512):
remove_file('./data/pictures/' + filename)
#raise web.HTTPBadRequest(reason='Cannot resize picture (status=%d, stdout="%s", stderr="%s")' % (status, stdout, stderr))
raise web.HTTPBadRequest(reason='Cannot resize picture')
remove_file('./data/pictures/' + orig_filename)
return '/pictures/' + filename
def run(self):
print "Starting capture" + self.dirname
old_path = None
while True:
threadLock.acquire()
path = cap_to_local_dir(self.cap)
if not path == None:
if old_path == None:
old_path = path
try:
if self.cap.imgcompare(old_path, path) > 0.97 and \
not old_path == path:
logger.debug("image is similar with old one, remove it")
util.remove_file(path)
else:
logger.debug("image saved to %s" % path)
old_path = path
except:
# work around after archive old_path is not exist
threadLock.release()
old_path = None
time.sleep(self.sleep_time)
continue
else:
logger.error("capture image error!")
old_path = None
threadLock.release()
time.sleep(self.sleep_time)
def swf2text(swf):
if re.search('\||\&|>', swf):
raise ValueError('Invalid swf file name')
tmp = '/tmp/' + str(uuid.uuid4()) + '.txt'
os.system('swfstrings %s >%s' % (swf, tmp))
text = ''
with open(tmp) as f:
for l in f:
text += l
remove_file(tmp)
return text
def cleanup(v):
"""
Renanme the output file and delete any temporary files.
"""
vmsg(v, 'Cleaning up')
# Remove specific files
util.remove_file(DEVICE_HDR)
# Remove runtime objects
for x in glob.glob('*.o'):
util.remove_file(x)
def main(cdl_ws, cdl_year='', overwrite_flag=False):
"""Download national CDL zips
Args:
cdl_ws (str): Folder/workspace path of the GIS data for the project
cdl_year (str): Comma separated list and/or range of years
overwrite_flag (bool): If True, overwrite existing files
Returns:
None
"""
logging.info('\nDownload and extract CONUS CDL rasters')
site_url = 'ftp://ftp.nass.usda.gov/download/res'
cdl_format = '{}_30m_cdls.{}'
for cdl_year in list(util.parse_int_set(cdl_year)):
logging.info('Year: {}'.format(cdl_year))
zip_name = cdl_format.format(cdl_year, 'zip')
zip_url = site_url + '/' + zip_name
zip_path = os.path.join(cdl_ws, zip_name)
cdl_path = os.path.join(cdl_ws, cdl_format.format(cdl_year, 'img'))
if not os.path.isdir(cdl_ws):
os.makedirs(cdl_ws)
if os.path.isfile(zip_path) and overwrite_flag:
os.remove(zip_path)
if not os.path.isfile(zip_path):
logging.info(' Download CDL files')
logging.debug(' {}'.format(zip_url))
logging.debug(' {}'.format(zip_path))
try:
urllib.urlretrieve(zip_url, zip_path)
except IOError as e:
logging.error(' IOError, skipping')
logging.error(e)
if os.path.isfile(cdl_path) and overwrite_flag:
util.remove_file(cdl_path)
if os.path.isfile(zip_path) and not os.path.isfile(cdl_path):
logging.info(' Extracting CDL files')
with zipfile.ZipFile(zip_path) as zf:
zf.extractall(cdl_ws)
async def new_user(request):
reader = await request.multipart()
data = {}
while True: # handle file upload and other fields
field = await reader.next()
if not field:
break
if field.name in ['email', 'name', 'password']:
data[field.name] = (await field.read(decode=True)).decode('utf8')
elif field.name == 'picture':
if field.filename == '': # skip if no file was provided
continue
data['picture'] = await upload_picture(field)
else:
raise web.HTTPBadRequest()
email = data.get('email', '')
password = data.get('password', '')
name = data.get('name', '')
picture = data.get('picture', '')
if picture.strip() == '':
picture = 'https://api.adorable.io/avatars/512/' + email
error = None
if 'email' == '' or password == '':
error = 'invalid request'
user_id = await users.add(password=password,
email=email,
name=name,
picture=picture)
if user_id is None:
error = 'email already exists'
if error is not None:
remove_file('./data/' + picture)
return {'email': email, 'name': name, 'error_message': error}
else:
session = await new_session(request)
session['user_id'] = str(user_id)
await history.add(user_id, 'create-user')
user = await users.get(user_id)
update_qrcode_image(user)
raise web.HTTPFound('/')
def run_test(self, name, path, num_cores, cmp_flags, param):
"""
Run a single test.
"""
try:
filename = path + '/' + name + '.sire'
# Substitute parameters in a new temporary file
if len(param) > 0:
s = read_file(filename)
for (var, value) in param:
p = re.compile('val {} is [0-9]+;'.format(var))
s = p.sub('val {} is {};'.format(var, value), s, count=1)
filename = os.getcwd() + '/' + name + '.sire.tmp'
write_file(filename, s)
# Compile the program
(exit, output) = call([COMPILE, filename] +
['-t', 'XS1', '-n', '{}'.format(num_cores)] +
cmp_flags)
self.assertTrue(exit)
# Delete the temporary version
if len(param) > 0:
remove_file(filename)
# Simulate execution
(exit, output) = call([SIMULATE, 'a.se'] + SIM_FLAGS)
self.assertTrue(exit)
# Check the output against the .output file
self.assertEqual(output.strip(),
read_file(path + '/' + name + '.output').strip())
except Exception as e:
sys.stderr.write('Error: {}\n'.format(e))
raise
def run_test(self, name, path, num_cores, cmp_flags, param):
"""
Run a single test.
"""
try:
filename = path+'/'+name+'.sire'
# Substitute parameters in a new temporary file
if len(param) > 0:
s = read_file(filename)
for (var, value) in param:
p = re.compile('val {} is [0-9]+;'.format(var))
s = p.sub('val {} is {};'.format(var, value), s, count=1)
filename = os.getcwd()+'/'+name+'.sire.tmp'
write_file(filename, s)
# Compile the program
(exit, output) = call([COMPILE, filename]
+ ['-t', 'XS1', '-n', '{}'.format(num_cores)] + cmp_flags)
self.assertTrue(exit)
# Delete the temporary version
if len(param) > 0:
remove_file(filename)
# Simulate execution
(exit, output) = call([SIMULATE, 'a.se'] + SIM_FLAGS)
self.assertTrue(exit)
# Check the output against the .output file
self.assertEqual(output.strip(),
read_file(path+'/'+name+'.output').strip())
except Exception as e:
sys.stderr.write('Error: {}\n'.format(e))
raise
def process_dir(files, only_add, delete, ignore_cue):
def add_file_to_itunes(filename, cue=None):
filename = os.path.abspath(filename)
print "Adding file %s to iTunes" % filename
run("osascript %s '%s'" % (ADD_SONG, filename))
if cue is not None:
name = os.path.splitext(os.path.basename(filename))[0]
run(
'osascript %(bin)s '
'"%(track)s" "%(artist)s" "%(album)s" "%(genre)s" "%(name)s" "%(track_count)d" "%(track_index)d" "%(year)s"'
% {
"bin": SET_TAG,
"track": name,
"artist": cue.artist,
"album": cue.album,
"genre": getattr(cue, "genre", ""),
"name": cue.getname(name),
"track_count": cue.track_count,
"track_index": cue.track_index(name),
"year": cue.year
})
extensions = [extract_extension(f) for f in files]
audio_files = filter(lambda f: extract_extension(f) in AUDIO_EXT, files)
audio_ext = set(filter(AUDIO_EXT.__contains__, extensions))
playlist_ext = filter(PLAYLIST_EXT.__contains__, extensions)
if len(audio_ext) != 1:
print "Error: directory with target files must contain "\
"only one type of audio files, there are %s." % str(audio_ext)
return
cue = None
files_to_add = []
if "cue" in playlist_ext and not ignore_cue:
if playlist_ext.count("cue") > 1:
print "Error: more than one cue files, ignoring this path."
return
cue = Cue(filter(lambda x: extract_extension(x) == "cue", files)[0])
if len(audio_files) != cue.track_count and len(audio_files) != 1:
print "Error: number of audio files differ from number of tracks, "\
"ignoring this path, %d %d " % (len(audio_files), cue.track_count)
return
if len(audio_files) == 1:
if extract_extension(audio_files[0]) not in ["flac", "ape", "wav"]:
print "Error: wrong extension %s. It is not supported with cue. "\
"Ignoring this path."
return
run("xld -c \"%s\" -f wav \"%s\"" % (cue.filename, audio_files[0]))
if delete:
remove_file(audio_files[0])
audio_files = [
f for f in os.listdir(".") if extract_extension(f) == "wav"
]
for f in audio_files:
if extract_extension(f) in ["mp3", "m4a"]:
files_to_add.append(f)
elif not only_add:
basename, _ = os.path.splitext(f)
target_file = basename + ".m4a"
run("ffmpeg -i \"%s\" -acodec alac \"%s\"" % (f, target_file))
files_to_add.append(target_file)
if delete:
remove_file(f)
for f in files_to_add:
add_file_to_itunes(f, cue)
async def do_reddit_video_download(bot, submission: Submission,
on_success: Callable[[BinaryIO],
Awaitable[None]],
on_failure: Callable[[], Awaitable[None]]):
# noinspection PyProtectedMember
headers = {
"User-Agent": bot.config.user_agent,
"Authorization": f"Bearer {bot.reddit._core._authorizer.access_token}",
}
async with aiohttp.ClientSession(headers=headers) as session:
mpd_url = submission.media["reddit_video"]["dash_url"]
async with session.get(mpd_url) as r:
mpd_body = await r.text()
audio_url, video_urls = get_urls_from_mpd(mpd_url, mpd_body)
# noinspection PyProtectedMember
fallback_url = urlparse(
submission.media["reddit_video"]["fallback_url"])._replace(
query=None).geturl()
if fallback_url != video_urls[0]:
video_urls.insert(0, fallback_url)
for video_url in video_urls:
if await get_video_approx_size(
session, video_url) > DiscordLimit.file_limit:
continue
try:
audio_filename = f"@videos/audio_{submission.id}.mp4"
video_filename = f"@videos/video_{submission.id}.mp4"
filename = f"@videos/{submission.id}.mp4"
async def get_audio():
async with session.get(audio_url) as resp:
async for data in resp.content.iter_any():
async with aiofiles.open(audio_filename,
"ba") as f:
await f.write(data)
async def get_video():
async with session.get(video_url) as resp:
async for data in resp.content.iter_any():
async with aiofiles.open(video_filename,
"ba") as f:
await f.write(data)
if audio_url:
await asyncio.gather(get_audio(), get_video())
else:
await get_video()
async def run_ffmpeg():
inputs = [ffmpeg.input(video_filename), ffmpeg.input(audio_filename)] \
if audio_url else [ffmpeg.input(video_filename)]
ffmpeg.output(
*inputs,
filename,
strict="-2",
loglevel="quiet",
).run()
await run_ffmpeg()
with open(filename, "rb") as file:
if os.path.getsize(file.name) <= DiscordLimit.file_limit:
await on_success(file)
return
finally:
remove_file(audio_filename)
remove_file(video_filename)
remove_file(filename)
await on_failure()
def process_dir(files, only_add, delete, ignore_cue):
def add_file_to_itunes(filename, cue=None):
filename = os.path.abspath(filename)
print "Adding file %s to iTunes" % filename
run("osascript %s '%s'" % (ADD_SONG, filename))
if cue is not None:
name = os.path.splitext(os.path.basename(filename))[0]
run('osascript %(bin)s '
'"%(track)s" "%(artist)s" "%(album)s" "%(genre)s" "%(name)s" "%(track_count)d" "%(track_index)d" "%(year)s"' %
{"bin": SET_TAG,
"track": name,
"artist": cue.artist,
"album": cue.album,
"genre": getattr(cue, "genre", ""),
"name": cue.getname(name),
"track_count": cue.track_count,
"track_index": cue.track_index(name),
"year": cue.year})
extensions = [extract_extension(f) for f in files]
audio_files = filter(lambda f: extract_extension(f) in AUDIO_EXT, files)
audio_ext = set(filter(AUDIO_EXT.__contains__, extensions))
playlist_ext = filter(PLAYLIST_EXT.__contains__, extensions)
if len(audio_ext) != 1:
print "Error: directory with target files must contain "\
"only one type of audio files, there are %s." % str(audio_ext)
return
cue = None
files_to_add = []
if "cue" in playlist_ext and not ignore_cue:
if playlist_ext.count("cue") > 1:
print "Error: more than one cue files, ignoring this path."
return
cue = Cue(filter(lambda x: extract_extension(x) == "cue", files)[0])
if len(audio_files) != cue.track_count and len(audio_files) != 1:
print "Error: number of audio files differ from number of tracks, "\
"ignoring this path, %d %d " % (len(audio_files), cue.track_count)
return
if len(audio_files) == 1:
if extract_extension(audio_files[0]) not in ["flac", "ape", "wav"]:
print "Error: wrong extension %s. It is not supported with cue. "\
"Ignoring this path."
return
run("xld -c \"%s\" -f wav \"%s\"" % (cue.filename, audio_files[0]))
if delete:
remove_file(audio_files[0])
audio_files = [f for f in os.listdir(".") if extract_extension(f) == "wav"]
for f in audio_files:
if extract_extension(f) in ["mp3", "m4a"]:
files_to_add.append(f)
elif not only_add:
basename, _ = os.path.splitext(f)
target_file = basename + ".m4a"
run("ffmpeg -i \"%s\" -acodec alac \"%s\"" % (f, target_file))
files_to_add.append(target_file)
if delete:
remove_file(f)
for f in files_to_add:
add_file_to_itunes(f, cue)
def splits(cls,
text_field,
label_field,
pair_field,
parse_field=None,
root='../../data',
train='train.csv',
validation='dev.csv',
test='test.csv'):
all_bad_datapoints = [
"2057991743.jpg#2r1c", "7638876050.jpg#0r1c",
"7638876050.jpg#4r2n", "7638876050.jpg#4r2e",
"7638876050.jpg#4r2c", "112178718.jpg#2r5c", "112178718.jpg#2r5n",
"507370108.jpg#4r1e", "507370108.jpg#4r1c", "1153704539.jpg#0r1c",
"3398745929.jpg#3r1c", "2753157857.jpg#2r4c", "651277216.jpg#1r1n",
"4882073197.jpg#2r3n", "4882073197.jpg#2r3c",
"3808935147.jpg#1r1c", "3808935147.jpg#1r1n",
"3808935147.jpg#1r1e", "162967671.jpg#0r2c", "2755595842.jpg#2r1e",
"4679327842.jpg#1r1c", "4679327842.jpg#1r1n",
"5062422406.jpg#3r1c", "5062422406.jpg#3r1e", "5062422406.jpg#3r1n"
]
empty_datapoints = [
"7638876050.jpg#4r2n", "7638876050.jpg#4r2e", '7638876050.jpg#4r2c'
]
path = cls.download(root, check='esnli')
my_args = util.get_args()
if my_args.sanity:
train = "dev.csv"
validation = "dev.csv"
test = "dev.csv"
else:
train = my_args.train_file
validation = my_args.dev_file
test = my_args.test_file
#for sanity checks we will give same train/dev/test and get the first n_data datapoints out, -1 stands for taking all
n_data = my_args.n_data
if train == validation and validation == test and n_data > -1:
print("Using smaller dataset")
# create a subset of the dataset with the n_data rows
f = open(path + "/" + train)
reader = csv.DictReader(f)
headers = reader.fieldnames
subset_dataset_file = path + "/" + str(n_data) + "subset_" + train
util.remove_file(subset_dataset_file)
g = open(subset_dataset_file, "a")
writer = csv.writer(g)
writer.writerow(headers)
i = 0
while i < n_data:
row = next(reader)
writer.writerow(row.values())
i += 1
train = str(n_data) + "subset_" + train
validation = str(n_data) + "subset_" + validation
test = str(n_data) + "subset_" + test
g.close()
f.close()
assert (parse_field is None)
return super(eSNLI,
cls).splits(path,
root,
train,
validation,
test,
format='csv',
fields=[('gold_label', label_field),
('sentence1_binary_parse', None),
('sentence2_binary_parse', None),
('sentence1_parse', None),
('sentence2_parse', None),
('sentence1', text_field),
('sentence2', text_field),
('captionID', None),
('pairID', pair_field),
('label1', None), ('label2', None),
('label3', None), ('label4', None),
('label5', None)],
filter_pred=lambda ex:
(ex.gold_label != '-') and
(ex.pairID not in empty_datapoints))
def process_dir(files, only_add, delete, ignore_cue):
def add_file_to_itunes(filename, cue=None):
filename = os.path.abspath(filename)
print "Adding file %s to iTunes" % filename
run('osascript', ADD_SONG, filename)
if cue is not None:
name = os.path.splitext(os.path.basename(filename))[0]
run('osascript', SET_TAG, name, cue.artist, cue.album, getattr(cue, 'genre'), cue.getname(name),
cue.track_count, cue.track_index(name), cue.year)
extensions = [extract_extension(f) for f in files]
audio_files = filter(lambda f: extract_extension(f) in AUDIO_EXT, files)
audio_ext = set(filter(AUDIO_EXT.__contains__, extensions))
playlist_ext = filter(PLAYLIST_EXT.__contains__, extensions)
if len(audio_ext) != 1:
print "Error: directory with target files must contain "\
"only one type of audio files, there are %s." % str(audio_ext)
return
cue = None
files_to_add = []
temp_files = []
if "cue" in playlist_ext and not ignore_cue:
if playlist_ext.count("cue") > 1:
print "Error: more than one cue files, ignoring this path."
return
cue = Cue(filter(lambda x: extract_extension(x) == "cue", files)[0])
if len(audio_files) != cue.track_count and len(audio_files) != 1:
print "Error: number of audio files differ from number of tracks, "\
"ignoring this path, %d %d " % (len(audio_files), cue.track_count)
return
if len(audio_files) == 1:
if extract_extension(audio_files[0]) not in ["flac", "ape", "wav"]:
print "Error: wrong extension %s. It is not supported with cue. "\
"Ignoring this path."
return
run('xld', '-c', cue.filename, '-f wav', audio_files[0])
if delete:
remove_file(audio_files[0])
audio_files = [f for f in os.listdir(".") if extract_extension(f) == "wav"]
for f in audio_files:
if extract_extension(f) in ["mp3", "m4a"]:
files_to_add.append(f)
elif not only_add:
basename, _ = os.path.splitext(f)
target_file = basename + ".m4a"
run('ffmpeg', '-i', f, '-acodec alac', target_file)
file_info = None
if extract_extension(f) == 'flac':
file_info = flac.FLAC(f)
elif extract_extension(f) == 'mp3':
file_info = mp3.MP3(f)
if file_info:
track_name = file_info.tags.get('ARTIST'), file_info.tags.get('ALBUM'), file_info.tags.get('TITLE')
if all(track_name):
track_name = tuple(n[0] for n in track_name)
track_info = TrackInfo(*track_name)
target_file_info = mp4.MP4(target_file)
if track_info.year:
target_file_info['\xa9day'] = str(track_info.year)
if track_info.track_index:
target_file_info['trkn'] = [(track_info.track_index, track_info.cd_track_count or 0)]
if track_info.cd_index:
target_file_info['disk'] = [(track_info.cd_index, track_info.cd_count or 0)]
if track_info.track_name:
target_file_info['\xa9nam'] = track_info.track_name
target_file_info.save()
if track_info.cover_art_file_name:
run('AtomicParsley', target_file, '--artwork', track_info.cover_art_file_name, '--overWrite')
files_to_add.append(target_file)
temp_files.append(target_file)
if delete:
remove_file(f)
for filename in files_to_add:
add_file_to_itunes(filename, cue)
for filename in temp_files:
remove_file(filename)
def main(gis_ws,
tile_ws,
dem_cs,
overwrite_flag=False,
pyramids_flag=False,
stats_flag=False):
"""Merge, project, and clip NED tiles
Args:
gis_ws (str): Folder/workspace path of the GIS data for the project
tile_ws (str): Folder/workspace path of the DEM tiles
dem_cs (int): DEM cellsize (10 or 30m)
overwrite_flag (bool): If True, overwrite existing files
pyramids_flag (bool): If True, build pyramids/overviews
for the output rasters
stats_flag (bool): If True, compute statistics for the output rasters
Returns:
None
"""
logging.info('\nPrepare DEM tiles')
# Inputs
output_units = 'METERS'
dem_ws = os.path.join(gis_ws, 'dem')
scratch_ws = os.path.join(gis_ws, 'scratch')
zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img')
# Use 1 degree snap point and "cellsize" to get 1x1 degree tiles
tile_osr = gdc.epsg_osr(4269)
tile_buffer = 0.5
tile_x, tile_y, tile_cs = 0, 0, 1
# Input error checking
if not os.path.isdir(gis_ws):
logging.error(('\nERROR: The GIS workspace {} ' +
'does not exist').format(gis_ws))
sys.exit()
elif not os.path.isdir(tile_ws):
logging.error(('\nERROR: The DEM tile workspace {} ' +
'does not exist').format(tile_ws))
sys.exit()
elif not os.path.isfile(zone_raster_path):
logging.error(('\nERROR: The zone raster {} does not exist' +
'\n Try re-running "build_study_area_raster.py"'
).format(zone_raster_path))
sys.exit()
elif output_units not in ['FEET', 'METERS']:
logging.error('\nERROR: The output units must be FEET or METERS\n')
sys.exit()
logging.info('\nGIS Workspace: {}'.format(gis_ws))
logging.info('DEM Workspace: {}'.format(dem_ws))
logging.info('Tile Workspace: {}\n'.format(tile_ws))
# Input folder/files
if dem_cs == 10:
tile_fmt = 'imgn{0:02d}w{1:03d}_13.img'
elif dem_cs == 30:
tile_fmt = 'imgn{0:02d}w{1:03d}_1.img'
# Output folder/files
if not os.path.isdir(dem_ws):
os.makedirs(dem_ws)
# Output file names
dem_fmt = 'ned_{0}m{1}.img'
# dem_gcs = dem_fmt.format(dem_cs, '_nad83_meters')
# dem_feet = dem_fmt.format(dem_cs, '_nad83_feet')
# dem_proj = dem_fmt.format(dem_cs, '_albers')
# dem_hs = dem_fmt.format(dem_cs, '_hs')
dem_gcs_path = os.path.join(dem_ws, dem_fmt.format(dem_cs,
'_nad83_meters'))
dem_feet_path = os.path.join(dem_ws, dem_fmt.format(dem_cs, '_nad83_feet'))
dem_proj_path = os.path.join(dem_ws, dem_fmt.format(dem_cs, '_albers'))
dem_hs_path = os.path.join(dem_ws, dem_fmt.format(dem_cs, '_hs'))
#
f32_nodata = float(np.finfo(np.float32).min)
if pyramids_flag:
levels = '2 4 8 16 32 64 128'
# gdal.SetConfigOption('USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_USE_RRD', 'YES')
# Reference all output rasters zone raster
zone_raster_ds = gdal.Open(zone_raster_path)
output_osr = gdc.raster_ds_osr(zone_raster_ds)
output_wkt = gdc.raster_ds_proj(zone_raster_ds)
output_cs = gdc.raster_ds_cellsize(zone_raster_ds)[0]
output_x, output_y = gdc.raster_ds_origin(zone_raster_ds)
output_extent = gdc.raster_ds_extent(zone_raster_ds)
zone_raster_ds = None
logging.debug('\nStudy area properties')
logging.debug(' Output OSR: {}'.format(output_osr))
logging.debug(' Output Extent: {}'.format(output_extent))
logging.debug(' Output cellsize: {}'.format(output_cs))
# Project study area extent to DEM tile coordinate system
tile_extent = gdc.project_extent(output_extent, output_osr, tile_osr)
logging.debug('Output Extent: {}'.format(tile_extent))
# Extent needed to select 1x1 degree tiles
tile_extent.buffer_extent(tile_buffer)
tile_extent.adjust_to_snap('EXPAND', tile_x, tile_y, tile_cs)
logging.debug('Tile Extent: {}'.format(tile_extent))
# Get list of available tiles that intersect the extent
input_path_list = sorted(
list(
set([
tile_fmt.format(lat, -lon)
# os.path.join(tile_ws, tile_fmt.format(lat, -lon))
for lon in range(int(tile_extent.xmin), int(tile_extent.xmax))
for lat in range(int(tile_extent.ymax), int(
tile_extent.ymin), -1) if os.path.isfile(
os.path.join(tile_ws, tile_fmt.format(lat, -lon)))
])))
logging.debug('Tiles')
# for input_path in input_path_list:
# .debug(' {}'.format(input_path))
# Calculate using GDAL utilities
if input_path_list:
logging.info('Merging tiles')
if os.path.isfile(dem_gcs_path) and overwrite_flag:
util.remove_file(dem_gcs_path)
# subprocess.call(
# 'gdalmanage', 'delete', '-f', 'HFA', dem_gcs_path])
if not os.path.isfile(dem_gcs_path):
# gdal_merge.py was only working if shell=True
# It would also work to add the scripts folder to the path (in Pythong)
# Or the scripts folder could be added to the system PYTHONPATH?
args_list = [
'python', '{}\scripts\gdal_merge.py'.format(
sys.exec_prefix), '-o', dem_gcs_path, '-of', 'HFA', '-co',
'COMPRESSED=YES', '-a_nodata',
str(f32_nodata)
] + input_path_list
logging.debug(args_list)
logging.debug('command length: {}'.format(len(
' '.join(args_list))))
subprocess.call(args_list, cwd=tile_ws)
# subprocess.call(
# 'set', 'GDAL_DATA={}\Lib\site-packages\osgeo\data\gdal'.format(sys.exec_prefix)],
# =True)
# subprocess.call(
# 'gdal_merge.py', '-o', dem_gcs_path, '-of', 'HFA',
# '-co', 'COMPRESSED=YES', '-a_nodata',
# str(f32_nodata)] + input_path_list,
# =True)
# Convert DEM from meters to feet
if output_units == 'FEET':
# DEADBEEF - This won't run when called through subprocess?
# subprocess.call(
# 'gdal_calc.py', '-A', dem_gcs_path,
# '--outfile={}'.format(dem_feet_path), '--calc="0.3048*A"',
# '--format', 'HFA', '--co', 'COMPRESSED=YES',
# '--NoDataValue={}'.format(str(f32_nodata)),
# '--type', 'Float32', '--overwrite'],
# =dem_ws, shell=True)
# dem_gcs_path = dem_feet_path
# Scale the values using custom function
m2ft_func(dem_gcs_path)
if os.path.isfile(dem_proj_path) and overwrite_flag:
subprocess.call(['gdalmanage', 'delete', '-f', 'HFA', dem_proj_path])
if os.path.isfile(dem_hs_path) and overwrite_flag:
subprocess.call(['gdalmanage', 'delete', '-f', 'HFA', dem_hs_path])
if (not os.path.isfile(dem_proj_path) and os.path.isfile(dem_gcs_path)):
subprocess.call([
'gdalwarp', '-r', 'bilinear', '-tr',
str(output_cs),
str(output_cs), '-s_srs', 'EPSG:4269', '-t_srs', output_wkt, '-ot',
'Float32'
] + ['-te'] + str(output_extent).split() +
# ['-srcnodata', 'None', '-dstnodata', str(f32_nodata),
[
'-of', 'HFA', '-co', 'COMPRESSED=YES',
'-overwrite', '-multi', '-wm', '1024', '-wo',
'NUM_THREADS=ALL_CPUS', dem_gcs_path, dem_proj_path
])
if (not os.path.isfile(dem_hs_path) and os.path.isfile(dem_proj_path)):
subprocess.call([
'gdaldem', 'hillshade', dem_proj_path, dem_hs_path, '-of', 'HFA',
'-co', 'COMPRESSED=YES'
])
if stats_flag:
logging.info('Computing statistics')
if os.path.isfile(dem_proj_path):
logging.debug(' {}'.format(dem_proj_path))
subprocess.call(['gdalinfo', '-stats', '-nomd', dem_proj_path])
if os.path.isfile(dem_hs_path):
logging.debug(' {}'.format(dem_hs_path))
subprocess.call(['gdalinfo', '-stats', '-nomd', dem_hs_path])
if pyramids_flag:
logging.info('\nBuilding pyramids')
if os.path.isfile(dem_proj_path):
logging.debug(' {}'.format(dem_proj_path))
subprocess.call(['gdaladdo', '-ro', dem_proj_path] +
levels.split())
if os.path.isfile(dem_hs_path):
logging.debug(' {}'.format(dem_hs_path))
subprocess.call(['gdaladdo', '-ro', dem_hs_path] + levels.split())
# subprocess.call(
# 'gdaladdo', '-ro', '--config', 'USE_RRD', 'YES',
# '--config', 'HFA_USE_RRD', 'YES', dem_proj_path] + levels.split()])
# subprocess.call(
# 'gdaladdo', '-ro', '--config', 'USE_RRD', 'YES',
# '--config', 'HFA_USE_RRD', 'YES', dem_hs_path] + levels.split()])
if os.path.isfile(os.path.join(dem_ws, dem_gcs_path)):
subprocess.call(['gdalmanage', 'delete', '-f', 'HFA', dem_gcs_path])
def main(gis_ws,
input_soil_ws,
prop_list=['all'],
overwrite_flag=False,
pyramids_flag=False,
stats_flag=False):
"""Convert soil polygon shapefiles to raster
Snap to latest CDL rasters (in CDL workspace) with an albers projection
Args:
gis_ws (str): Folder/workspace path of the GIS data for the project
input_soil_ws (str): Folder/workspace path of the common soils data
prop_list (list): String of the soil types to build
(i.e. awc, clay, sand, all)
overwrite_flag (bool): If True, overwrite output rasters
pyramids_flag (bool): If True, build pyramids/overviews
for the output rasters
stats_flag (bool): If True, compute statistics for the output rasters
Returns:
None
"""
logging.info('\nRasterizing Soil Polygons')
folder_fmt = 'gsmsoil_{}'
polygon_fmt = 'gsmsoilmu_a_us_{}_albers.shp'
output_soil_ws = os.path.join(gis_ws, 'soils')
scratch_ws = os.path.join(gis_ws, 'scratch')
zone_raster_path = os.path.join(scratch_ws, 'zone_raster.img')
# Soil polygons have a float and integer field
field_fmt = '{}'
# field_fmt = '{}_INT'
raster_fmt = '{}_30m_albers.img'
# raster_fmt = '{}_2010_30m_cdls.img'
# raster_fmt = 'gsmsoil_{}_integer.img'
output_format = 'HFA'
output_type = 'Float32'
output_nodata = float(np.finfo(np.float32).min)
# output_type = 'Byte'
# output_nodata = 255
if pyramids_flag:
levels = '2 4 8 16 32 64 128'
# gdal.SetConfigOption('USE_RRD', 'YES')
# gdal.SetConfigOption('HFA_USE_RRD', 'YES')
logging.info('Soil Property: {}'.format(', '.join(prop_list)))
if prop_list == ['all']:
prop_list = ['awc', 'clay', 'sand']
# Check input folders
if not os.path.isdir(gis_ws):
logging.error('\nERROR: The GIS workspace {} ' +
'does not exist\n'.format(gis_ws))
sys.exit()
elif not os.path.isdir(input_soil_ws):
logging.error(('\nERROR: The input soil workspace {} ' +
'does not exist').format(input_soil_ws))
sys.exit()
elif not os.path.isfile(zone_raster_path):
logging.error(('\nERROR: The zone raster {} does not exist' +
'\n Try re-running "build_study_area_raster.py"'
).format(zone_raster_path))
sys.exit()
if not os.path.isdir(output_soil_ws):
os.makedirs(output_soil_ws)
logging.info('\nGIS Workspace: {}'.format(gis_ws))
logging.info('Soil Workspace: {}\n'.format(output_soil_ws))
temp_polygon_path = os.path.join(output_soil_ws, 'temp_polygon.shp')
if os.path.isfile(temp_polygon_path):
util.remove_file(temp_polygon_path)
# subprocess.call(
# ['gdalmanage', 'delete', '-f', '', temp_polygon_path])
# Reference all output rasters zone raster
zone_raster_ds = gdal.Open(zone_raster_path)
output_osr = gdc.raster_ds_osr(zone_raster_ds)
output_wkt = gdc.raster_ds_proj(zone_raster_ds)
output_cs = gdc.raster_ds_cellsize(zone_raster_ds)[0]
output_x, output_y = gdc.raster_ds_origin(zone_raster_ds)
output_extent = gdc.raster_ds_extent(zone_raster_ds)
zone_raster_ds = None
logging.debug('\nStudy area properties')
logging.debug(' Output OSR: {}'.format(output_osr))
logging.debug(' Output Extent: {}'.format(output_extent))
logging.debug(' Output cellsize: {}'.format(output_cs))
# Process each soil property
for prop_str in prop_list:
input_polygon_path = os.path.join(input_soil_ws,
folder_fmt.format(prop_str),
polygon_fmt.format(prop_str))
output_raster_path = os.path.join(output_soil_ws,
raster_fmt.format(prop_str))
if not os.path.isfile(input_polygon_path):
logging.info(('The soil polygon {} does not ' +
'exist').format(input_polygon_path))
continue
elif os.path.isfile(output_raster_path) and overwrite_flag:
subprocess.call(['gdalmanage', 'delete', output_raster_path])
if not os.path.isfile(output_raster_path):
soil_field = field_fmt.format(prop_str.upper())
logging.info('Projecting shapefile')
# Project study area extent to the input/soil spatial reference
input_osr = gdc.feature_path_osr(input_polygon_path)
input_extent = gdc.project_extent(output_extent, output_osr,
input_osr)
logging.debug('Input Extent: {}'.format(input_extent))
subprocess.call([
'ogr2ogr', '-overwrite', '-preserve_fid', '-t_srs',
str(output_wkt), '-spat',
str(input_extent.xmin),
str(input_extent.ymin),
str(input_extent.ymax),
str(input_extent.ymax), temp_polygon_path, input_polygon_path
])
logging.info('Rasterizing shapefile')
subprocess.call([
'gdal_rasterize', '-of', output_format, '-a', soil_field,
'-a_nodata',
str(output_nodata), '-init',
str(output_nodata), '-co', 'COMPRESSED=YES'
] + ['-te'] + str(output_extent).split() + [
'-tr',
str(output_cs),
str(output_cs), '-ot', output_type, temp_polygon_path,
output_raster_path
])
if os.path.isfile(temp_polygon_path):
util.remove_file(temp_polygon_path)
# subprocess.call(['gdalmanage', 'delete', temp_polygon_path])
if stats_flag and os.path.isfile(output_raster_path):
logging.info('Computing statistics')
logging.debug(' {}'.format(output_raster_path))
subprocess.call(
['gdalinfo', '-stats', '-nomd', output_raster_path])
if pyramids_flag and os.path.isfile(output_raster_path):
logging.info('Building pyramids')
logging.debug(' {}'.format(output_raster_path))
subprocess.call(['gdaladdo', '-ro', output_raster_path] +
levels.split())
def cleanup(v):
"""
Renanme the output file and delete any temporary files.
"""
vmsg(v, 'Cleaning up')
# Remove specific files
util.remove_file(MASTER_XE)
util.remove_file(SLAVE_XE)
util.remove_file('image_n0c0.elf')
util.remove_file('config.xml')
util.remove_file('platform_def.xn')
util.remove_file('program_info.txt')
util.remove_file(DEVICE_HDR)
# Remove unused master images
for x in glob.glob('image_n*c*elf'):
util.remove_file(x)
# Remove runtime objects
for x in glob.glob('*.o'):
util.remove_file(x)
def text_recognition(path, config):
root, ext = os.path.splitext(path)
txt_path = root + ".txt"
if os.path.exists(txt_path):
with open(txt_path) as f:
out = json.loads(open(txt_path).read())
return out
from google.cloud import speech
from google.cloud.speech import enums
from google.cloud.speech import types
out = {}
error_count = 0
tmp_path = os.path.splitext(path)[0] + ".tmp.wav"
client = speech.SpeechClient() # Fixed
while True:
try:
# client= speech.SpeechClient() # Causes 10060 max retries exceeded -to OAuth -HK
content = load_audio(
path,
pre_silence_length=config.pre_silence_length,
post_silence_length=config.post_silence_length)
max_duration = config.max_duration - \
config.pre_silence_length - config.post_silence_length
audio_duration = get_duration(content)
if audio_duration >= max_duration:
print(" [!] Skip {} because of duration: {} > {}". \
format(path, audio_duration, max_duration))
return {}
content = resample_audio(content, config.sample_rate)
save_audio(content, tmp_path, config.sample_rate)
with io.open(tmp_path, 'rb') as f:
audio = types.RecognitionAudio(content=f.read())
config = types.RecognitionConfig(
encoding=enums.RecognitionConfig.AudioEncoding.LINEAR16,
sample_rate_hertz=config.sample_rate,
language_code='en-US')
response = client.recognize(config, audio)
if len(response.results) > 0:
alternatives = response.results[0].alternatives
results = [
alternative.transcript for alternative in alternatives
]
assert len(results) == 1, "More than 1 results: {}".format(
results)
out = {path: "" if len(results) == 0 else results[0]}
print(path, results[0])
break
break
except Exception as err:
raise Exception("OS error: {0}".format(err))
error_count += 1
print("Skip warning for {} for {} times". \
format(path, error_count))
if error_count > 5:
break
else:
continue
remove_file(tmp_path)
with open(txt_path, 'w') as f:
json.dump(out, f, indent=2, ensure_ascii=False)
return out
results.update(item)
return results
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--audio_pattern', required=True)
parser.add_argument('--recognition_filename', default="recognition.json")
parser.add_argument('--sample_rate', default=16000, type=int)
parser.add_argument('--pre_silence_length', default=1, type=int)
parser.add_argument('--post_silence_length', default=1, type=int)
parser.add_argument('--max_duration', default=60, type=int)
config, unparsed = parser.parse_known_args()
audio_dir = os.path.dirname(config.audio_pattern)
for tmp_path in glob(os.path.join(audio_dir, "*.tmp.*")):
remove_file(tmp_path)
paths = glob(config.audio_pattern)
paths.sort()
results = text_recognition_batch(paths, config)
base_dir = os.path.dirname(audio_dir)
recognition_path = \
os.path.join(base_dir, config.recognition_filename)
if os.path.exists(recognition_path):
backup_file(recognition_path)
write_json(recognition_path, results)