def _plot_target_network_rel_residuals(df,
target,
ref,
batch_options,
filter_options,
tt_scale=50,
snr_scale=(0, 60),
annotator=None):
file_label = batch_options.batch_label
save_file = batch_options.save_file
def _plot_dataset(ds, net_code, ref_code):
# Sort ds rows by SNR, so that the weakest SNR points are drawn first and the high SNR point last,
# to make sure high SNR point are in the top rendering layer.
ds = ds.sort_values('snr')
times = pandas_timestamp_to_plottable_datetime(ds['originTimestamp'])
vals = ds[yaxis].values
qual = ds['snr'].values
min_mag = 4.0
mag = ds['mag'].values - min_mag
ylabel = 'Relative TT residual (sec)'
title = r"Station {} TT residuals relative to network {} (filtering: ref SNR$\geq${}, CWT$\geq${}, "\
r"slope$\geq${}, $n\sigma\geq{}$)".format(ref_code, net_code, str(filter_options.min_event_snr),
str(filter_options.cwt_cutoff),
str(filter_options.slope_cutoff),
str(filter_options.nsigma_cutoff))
if vals.any():
plt.figure(figsize=(32, 9))
sc = plt.scatter(times,
vals,
c=qual,
alpha=0.5,
cmap='gnuplot_r',
s=np.maximum(50 * mag, 10),
edgecolors=None,
linewidths=0)
time_formatter = matplotlib.dates.DateFormatter("%Y-%m-%d")
plt.axes().xaxis.set_major_formatter(time_formatter)
cb = plt.colorbar(sc, drawedges=False)
cb.set_label('Signal to noise ratio', fontsize=12)
plt.grid(color='#808080', linestyle=':', alpha=0.7)
plt.xlabel(xlabel, fontsize=14)
plt.ylabel(ylabel, fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.xlim(time_range)
plt.ylim((-tt_scale, tt_scale))
plt.clim(snr_scale)
plt.title(title, fontsize=18)
plt.legend(['Point size = Mag - {}, Color = SNR'.format(min_mag)],
fontsize=12,
loc=1)
plt.text(0.01,
0.96,
"Channel selection: {}".format(
filter_options.channel_preference),
transform=plt.gca().transAxes,
fontsize=12)
plt.text(0.01,
0.92,
"Start date: {}".format(str(time_range[0])),
transform=plt.gca().transAxes,
fontsize=12)
plt.text(0.01,
0.88,
" End date: {}".format(str(time_range[1])),
transform=plt.gca().transAxes,
fontsize=12)
plt.tight_layout(pad=1.05)
if annotator is not None:
annotator()
if save_file:
subfolder = os.path.join(
ref_code.split('.')[0] + file_label, net_code)
# subfolder = net_code
pathlib.Path(subfolder).mkdir(exist_ok=True, parents=True)
plt_file = os.path.join(
subfolder, '_'.join([ref_code, net_code]) + '_' +
ylabel.replace(" ", "").replace(".*", "") + ".png")
plt.savefig(plt_file, dpi=150)
plt.close()
else: # pragma: no cover
plt.show()
# end plot_dataset
df_times = pandas_timestamp_to_plottable_datetime(df['originTimestamp'])
if batch_options.x_range is not None:
time_range = batch_options.x_range
else:
time_range = (df_times.min(), df_times.max())
ref_code = ".".join([ref['net'][0], ref['sta'][0]])
log = logging.getLogger(__name__)
log.info("Plotting time range " +
" to ".join([t.strftime("%Y-%m-%d %H:%M:%S")
for t in time_range]) + " for " + ref_code)
yaxis = 'relTtResidual'
xlabel = 'Event Origin Timestamp'
# Remove reference station from target set before producing composite image.
# The reference station may not be there, but remove it if it is.
mask_ref = compute_matching_network_mask(df, ref)
mask_targ = compute_matching_network_mask(df, target)
df_agg = df[(mask_targ) & (~mask_ref)]
_plot_dataset(df_agg, ','.join(np.unique(target['net'])), ref_code)
# Export median error for each origin event if export path is provided
if batch_options.export_path is not None:
pathlib.Path(batch_options.export_path).mkdir(parents=True,
exist_ok=True)
df_agg = df_agg.sort_values('originTimestamp')
df_export = df_agg[['originTimestamp', 'relTtResidual']]
median_errors = {'originTimestamp': [], 'rawClockError': []}
for origin_ts, df_event in df_export.groupby('originTimestamp'):
median_errors['originTimestamp'].append(origin_ts)
median_errors['rawClockError'].append(
df_event['relTtResidual'].median())
df_export = pd.DataFrame(median_errors)
fname = ref_code + ".raw_clock_error.csv"
fname = os.path.join(batch_options.export_path, fname)
df_export.to_csv(fname, index=False)
Config = {
'videomode': 0,
'stereomode': 0,
'nomipmap': 0,
'fixedtime': 0,
'sizewindow': 2,
'sizefullscreen': 0,
'bitdepth': 2,
'refreshrate': 2,
'pathplayer': 'engine/blenderplayer.exe',
'pathfile': 'data/main.blend'
}
if True:
cur_dir = os.path.basename(pathlib2.Path('').resolve().as_posix())
if cur_dir == 'bglauncher':
if os.name == 'nt':
Config['pathplayer'] = 'dist_win/engine/blenderplayer.exe'
Config['pathfile'] = 'dist_win/data/testfile.blend'
if os.name == 'posix':
Config['pathplayer'] = 'dist_linux/engine/blenderplayer'
Config['pathfile'] = 'dist_linux/data/testfile.blend'
if cur_dir == 'dist_win':
Config['pathplayer'] = 'engine/blenderplayer.exe'
Config['pathfile'] = 'data/testfile.blend'
def read_downsample_equalize_mask(impath, *args, **kwargs):
uri_impath = [pathlib.Path(fn).as_uri() for fn in impath]
return read_downsample_equalize_mask_uri(uri_impath, *args, **kwargs)
def main(self):
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--device-model-id',
'--device_model_id',
type=str,
metavar='DEVICE_MODEL_ID',
required=False,
help='the device model ID registered with Google')
parser.add_argument('--project-id',
'--project_id',
type=str,
metavar='PROJECT_ID',
required=False,
help='the project ID used to register this device')
parser.add_argument('--device-config',
type=str,
metavar='DEVICE_CONFIG_FILE',
default=os.path.join(
os.path.expanduser('~/.config'),
'googlesamples-assistant',
'device_config_library.json'),
help='path to store and read device configuration')
parser.add_argument('--credentials',
type=existing_file,
metavar='OAUTH2_CREDENTIALS_FILE',
default=os.path.join(
os.path.expanduser('~/.config'),
'google-oauthlib-tool', 'credentials.json'),
help='path to store and read OAuth2 credentials')
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s ' + Assistant.__version_str__())
args = parser.parse_args()
with open(args.credentials, 'r') as f:
credentials = google.oauth2.credentials.Credentials(token=None,
**json.load(f))
device_model_id = None
last_device_id = None
try:
with open(args.device_config) as f:
device_config = json.load(f)
device_model_id = device_config['model_id']
last_device_id = device_config.get('last_device_id', None)
except FileNotFoundError:
pass
if not args.device_model_id and not device_model_id:
raise Exception('Missing --device-model-id option')
# Re-register if "device_model_id" is given by the user and it differs
# from what we previously registered with.
should_register = (args.device_model_id
and args.device_model_id != device_model_id)
device_model_id = args.device_model_id or device_model_id
with Assistant(credentials, device_model_id) as assistant:
self.assistant = assistant
subprocess.Popen([
"aplay", "{}/sample-audio-files/Startup.wav".format(ROOT_PATH)
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
events = assistant.start()
device_id = assistant.device_id
print('device_model_id:', device_model_id)
print('device_id:', device_id + '\n')
# Re-register if "device_id" is different from the last "device_id":
if should_register or (device_id != last_device_id):
if args.project_id:
register_device(args.project_id, credentials,
device_model_id, device_id)
pathlib.Path(os.path.dirname(
args.device_config)).mkdir(exist_ok=True)
with open(args.device_config, 'w') as f:
json.dump(
{
'last_device_id': device_id,
'model_id': device_model_id,
}, f)
else:
print(WARNING_NOT_REGISTERED)
for event in events:
self.process_event(event)
usrcmd = event.args
if configuration['DIYHUE']['DIYHUE_Control'] == 'Enabled':
if os.path.isfile('/opt/hue-emulator/config.json'):
with open('/opt/hue-emulator/config.json',
'r') as config:
hueconfig = json.load(config)
for i in range(1, len(hueconfig['lights']) + 1):
try:
if str(
hueconfig['lights'][str(i)]
['name']).lower() in str(usrcmd).lower():
assistant.stop_conversation()
hue_control(
str(usrcmd).lower(), str(i),
str(hueconfig['lights_address'][str(i)]
['ip']))
break
except Keyerror:
say('Unable to help, please check your config file'
)
if configuration['Tasmota_devicelist'][
'Tasmota_Control'] == 'Enabled':
for num, name in enumerate(tasmota_devicelist):
if name.lower() in str(usrcmd).lower():
assistant.stop_conversation()
tasmota_control(
str(usrcmd).lower(), name.lower(),
tasmota_deviceip[num],
tasmota_deviceportid[num])
break
if configuration['Conversation'][
'Conversation_Control'] == 'Enabled':
for i in range(1, numques + 1):
try:
if str(configuration['Conversation']['question'][i]
[0]).lower() in str(usrcmd).lower():
assistant.stop_conversation()
selectedans = random.sample(
configuration['Conversation']['answer'][i],
1)
say(selectedans[0])
break
except Keyerror:
say('Please check if the number of questions matches the number of answers'
)
if Domoticz_Device_Control == True and len(
domoticz_devices['result']) > 0:
if len(configuration['Domoticz']['Devices']
['Name']) == len(
configuration['Domoticz']['Devices']['Id']):
for i in range(
0,
len(configuration['Domoticz']['Devices']
['Name'])):
if str(configuration['Domoticz']['Devices']['Name']
[i]).lower() in str(usrcmd).lower():
assistant.stop_conversation()
domoticz_control(
str(usrcmd).lower(),
configuration['Domoticz']['Devices']['Id']
[i], configuration['Domoticz']['Devices']
['Name'][i])
break
else:
say("Number of devices and the number of ids given in config file do not match"
)
if (custom_action_keyword['Keywords']['Magic_mirror'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
try:
mmmcommand = str(usrcmd).lower()
if 'weather'.lower() in mmmcommand:
if 'show'.lower() in mmmcommand:
mmreq_one = requests.get(
"http://" + mmmip +
":8080/remote?action=SHOW&module=module_2_currentweather"
)
mmreq_two = requests.get(
"http://" + mmmip +
":8080/remote?action=SHOW&module=module_3_currentweather"
)
if 'hide'.lower() in mmmcommand:
mmreq_one = requests.get(
"http://" + mmmip +
":8080/remote?action=HIDE&module=module_2_currentweather"
)
mmreq_two = requests.get(
"http://" + mmmip +
":8080/remote?action=HIDE&module=module_3_currentweather"
)
if 'power off'.lower() in mmmcommand:
mmreq = requests.get(
"http://" + mmmip +
":8080/remote?action=SHUTDOWN")
if 'reboot'.lower() in mmmcommand:
mmreq = requests.get("http://" + mmmip +
":8080/remote?action=REBOOT")
if 'restart'.lower() in mmmcommand:
mmreq = requests.get("http://" + mmmip +
":8080/remote?action=RESTART")
if 'display on'.lower() in mmmcommand:
mmreq = requests.get(
"http://" + mmmip +
":8080/remote?action=MONITORON")
if 'display off'.lower() in mmmcommand:
mmreq = requests.get(
"http://" + mmmip +
":8080/remote?action=MONITOROFF")
except requests.exceptions.ConnectionError:
say("Magic mirror not online")
if (custom_action_keyword['Keywords']['Recipe_pushbullet'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
ingrequest = str(usrcmd).lower()
ingredientsidx = ingrequest.find('for')
ingrequest = ingrequest[ingredientsidx:]
ingrequest = ingrequest.replace('for', "", 1)
ingrequest = ingrequest.replace("'}", "", 1)
ingrequest = ingrequest.strip()
ingrequest = ingrequest.replace(" ", "%20", 1)
getrecipe(ingrequest)
if (custom_action_keyword['Keywords']['Kickstarter_tracking']
[0]).lower() in str(usrcmd).lower():
assistant.stop_conversation()
kickstarter_tracker(str(usrcmd).lower())
if configuration['Raspberrypi_GPIO_Control'][
'GPIO_Control'] == 'Enabled':
if (custom_action_keyword['Keywords']['Pi_GPIO_control'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
Action(str(usrcmd).lower())
if configuration['YouTube']['YouTube_Control'] == 'Enabled':
if (custom_action_keyword['Keywords']
['YouTube_music_stream'][0]
).lower() in str(usrcmd).lower() and 'kodi' not in str(
usrcmd).lower() and 'chromecast' not in str(
usrcmd).lower():
assistant.stop_conversation()
vlcplayer.stop_vlc()
if 'autoplay'.lower() in str(usrcmd).lower():
YouTube_Autoplay(str(usrcmd).lower())
else:
YouTube_No_Autoplay(str(usrcmd).lower())
if (custom_action_keyword['Keywords']['Stop_music'][0]
).lower() in str(usrcmd).lower():
stop()
if configuration['Radio_stations'][
'Radio_Control'] == 'Enabled':
if 'radio'.lower() in str(usrcmd).lower():
assistant.stop_conversation()
radio(str(usrcmd).lower())
if configuration['ESP']['ESP_Control'] == 'Enabled':
if (custom_action_keyword['Keywords']['ESP_control'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
ESP(str(usrcmd).lower())
if (custom_action_keyword['Keywords']['Parcel_tracking'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
track()
if (custom_action_keyword['Keywords']['RSS'][0]
).lower() in str(usrcmd).lower() or (
custom_action_keyword['Keywords']['RSS'][1]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
feed(str(usrcmd).lower())
if kodicontrol:
if (custom_action_keyword['Keywords']['Kodi_actions'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
kodiactions(str(usrcmd).lower())
# Google Assistant now comes built in with chromecast control, so custom function has been commented
# if 'chromecast'.lower() in str(usrcmd).lower():
# assistant.stop_conversation()
# if 'play'.lower() in str(usrcmd).lower():
# chromecast_play_video(str(usrcmd).lower())
# else:
# chromecast_control(usrcmd)
if (custom_action_keyword['Keywords']['Pause_resume'][0]
).lower() in str(usrcmd).lower() or (
custom_action_keyword['Keywords']['Pause_resume'][1]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
if vlcplayer.is_vlc_playing():
if (custom_action_keyword['Keywords']['Pause_resume']
[0]).lower() in str(usrcmd).lower():
vlcplayer.pause_vlc()
if checkvlcpaused():
if (custom_action_keyword['Keywords']['Pause_resume']
[1]).lower() in str(usrcmd).lower():
vlcplayer.play_vlc()
elif vlcplayer.is_vlc_playing(
) == False and checkvlcpaused() == False:
say("Sorry nothing is playing right now")
if (custom_action_keyword['Keywords']['Track_change']['Next']
[0]).lower() in str(usrcmd).lower() or (
custom_action_keyword['Keywords']['Track_change']
['Next'][1]).lower() in str(usrcmd).lower() or (
custom_action_keyword['Keywords']['Track_change']
['Next'][2]).lower() in str(usrcmd).lower():
assistant.stop_conversation()
if vlcplayer.is_vlc_playing() or checkvlcpaused() == True:
vlcplayer.stop_vlc()
vlcplayer.change_media_next()
elif vlcplayer.is_vlc_playing(
) == False and checkvlcpaused() == False:
say("Sorry nothing is playing right now")
if (custom_action_keyword['Keywords']['Track_change']
['Previous'][0]).lower() in str(usrcmd).lower() or (
custom_action_keyword['Keywords']['Track_change']
['Previous'][1]).lower() in str(usrcmd).lower() or (
custom_action_keyword['Keywords']['Track_change']
['Previous'][2]).lower() in str(usrcmd).lower():
assistant.stop_conversation()
if vlcplayer.is_vlc_playing() or checkvlcpaused() == True:
vlcplayer.stop_vlc()
vlcplayer.change_media_previous()
elif vlcplayer.is_vlc_playing(
) == False and checkvlcpaused() == False:
say("Sorry nothing is playing right now")
if (custom_action_keyword['Keywords']['VLC_music_volume'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
if vlcplayer.is_vlc_playing() == True or checkvlcpaused(
) == True:
if (custom_action_keyword['Dict']['Set']
).lower() in str(usrcmd).lower() or (
custom_action_keyword['Dict']['Change']
).lower() in str(usrcmd).lower():
if 'hundred'.lower() in str(usrcmd).lower(
) or custom_action_keyword['Dict'][
'Maximum'] in str(usrcmd).lower():
settingvollevel = 100
with open(
'{}/.mediavolume.json'.format(
USER_PATH), 'w') as vol:
json.dump(settingvollevel, vol)
elif 'zero'.lower() in str(usrcmd).lower(
) or custom_action_keyword['Dict'][
'Minimum'] in str(usrcmd).lower():
settingvollevel = 0
with open(
'{}/.mediavolume.json'.format(
USER_PATH), 'w') as vol:
json.dump(settingvollevel, vol)
else:
for settingvollevel in re.findall(
r"[-+]?\d*\.\d+|\d+", str(usrcmd)):
with open(
'{}/.mediavolume.json'.format(
USER_PATH), 'w') as vol:
json.dump(settingvollevel, vol)
print('Setting volume to: ' + str(settingvollevel))
vlcplayer.set_vlc_volume(int(settingvollevel))
elif (custom_action_keyword['Dict']['Increase']
).lower() in str(usrcmd).lower() or (
custom_action_keyword['Dict']['Decrease']
).lower() in str(usrcmd).lower(
) or 'reduce'.lower() in str(usrcmd).lower():
if os.path.isfile(
"{}/.mediavolume.json".format(USER_PATH)):
with open(
'{}/.mediavolume.json'.format(
USER_PATH), 'r') as vol:
oldvollevel = json.load(vol)
for oldvollevel in re.findall(
r'\b\d+\b', str(oldvollevel)):
oldvollevel = int(oldvollevel)
else:
oldvollevel = vlcplayer.get_vlc_volume
for oldvollevel in re.findall(
r"[-+]?\d*\.\d+|\d+", str(output)):
oldvollevel = int(oldvollevel)
if (custom_action_keyword['Dict']['Increase']
).lower() in str(usrcmd).lower():
if any(char.isdigit() for char in str(usrcmd)):
for changevollevel in re.findall(
r'\b\d+\b', str(usrcmd)):
changevollevel = int(changevollevel)
else:
changevollevel = 10
newvollevel = oldvollevel + changevollevel
print(newvollevel)
if int(newvollevel) > 100:
settingvollevel = 100
elif int(newvollevel) < 0:
settingvollevel = 0
else:
settingvollevel = newvollevel
with open(
'{}/.mediavolume.json'.format(
USER_PATH), 'w') as vol:
json.dump(settingvollevel, vol)
print('Setting volume to: ' +
str(settingvollevel))
vlcplayer.set_vlc_volume(int(settingvollevel))
if (custom_action_keyword['Dict']['Decrease']
).lower() in str(usrcmd).lower(
) or 'reduce'.lower() in str(usrcmd).lower():
if any(char.isdigit() for char in str(usrcmd)):
for changevollevel in re.findall(
r'\b\d+\b', str(usrcmd)):
changevollevel = int(changevollevel)
else:
changevollevel = 10
newvollevel = oldvollevel - changevollevel
print(newvollevel)
if int(newvollevel) > 100:
settingvollevel = 100
elif int(newvollevel) < 0:
settingvollevel = 0
else:
settingvollevel = newvollevel
with open(
'{}/.mediavolume.json'.format(
USER_PATH), 'w') as vol:
json.dump(settingvollevel, vol)
print('Setting volume to: ' +
str(settingvollevel))
vlcplayer.set_vlc_volume(int(settingvollevel))
else:
say("Sorry I could not help you")
else:
say("Sorry nothing is playing right now")
if (custom_action_keyword['Keywords']['Music_index_refresh'][0]
).lower() in str(usrcmd).lower() and (
custom_action_keyword['Keywords']
['Music_index_refresh'][1]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
refreshlists()
if configuration['Gmusicapi']['Gmusic_Control'] == 'Enabled':
if (custom_action_keyword['Keywords']
['Google_music_streaming'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
vlcplayer.stop_vlc()
gmusicselect(str(usrcmd).lower())
if configuration['Spotify']['Spotify_Control'] == 'Enabled':
if (custom_action_keyword['Keywords']
['Spotify_music_streaming'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
vlcplayer.stop_vlc()
spotify_playlist_select(str(usrcmd).lower())
if configuration['Gaana']['Gaana_Control'] == 'Enabled':
if (custom_action_keyword['Keywords']
['Gaana_music_streaming'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
vlcplayer.stop_vlc()
gaana_playlist_select(str(usrcmd).lower())
if configuration['Deezer']['Deezer_Control'] == 'Enabled':
if (custom_action_keyword['Keywords']
['Deezer_music_streaming'][0]
).lower() in str(usrcmd).lower():
assistant.stop_conversation()
vlcplayer.stop_vlc()
deezer_playlist_select(str(usrcmd).lower())
if custom_wakeword:
self.detector.terminate()
def path_as_uri(path):
return pathlib.Path(os.path.abspath(path)).as_uri()
def test_py7zr_is_7zfile_path():
assert py7zr.is_7zfile(pathlib.Path(testdata_path).joinpath('test_1.7z'))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--device-model-id', '--device_model_id', type=str,
metavar='DEVICE_MODEL_ID', required=False,
help='the device model ID registered with Google')
parser.add_argument('--project-id', '--project_id', type=str,
metavar='PROJECT_ID', required=False,
help='the project ID used to register this device')
parser.add_argument('--nickname', type=str,
metavar='NICKNAME', required=False,
help='the nickname used to register this device')
parser.add_argument('--device-config', type=str,
metavar='DEVICE_CONFIG_FILE',
default=os.path.join(
os.path.expanduser('~/.config'),
'googlesamples-assistant',
'device_config_library.json'
),
help='path to store and read device configuration')
parser.add_argument('--credentials', type=existing_file,
metavar='OAUTH2_CREDENTIALS_FILE',
default=os.path.join(
os.path.expanduser('~/.config'),
'google-oauthlib-tool',
'credentials.json'
),
help='path to store and read OAuth2 credentials')
parser.add_argument('--query', type=str,
metavar='QUERY',
help='query to send as soon as the Assistant starts')
parser.add_argument('-v', '--version', action='version',
version='%(prog)s ' + Assistant.__version_str__())
args = parser.parse_args()
with open(args.credentials, 'r') as f:
credentials = google.oauth2.credentials.Credentials(token=None,
**json.load(f))
device_model_id = None
last_device_id = None
try:
with open(args.device_config) as f:
device_config = json.load(f)
device_model_id = device_config['model_id']
last_device_id = device_config.get('last_device_id', None)
except FileNotFoundError:
pass
if not args.device_model_id and not device_model_id:
raise Exception('Missing --device-model-id option')
should_register = (
args.device_model_id and args.device_model_id != device_model_id)
device_model_id = args.device_model_id or device_model_id
note = Notify.Notification.new(Notify.get_app_name(),'Micno with Assistant は動作中です','mic-volume-high')
note.set_urgency(Notify.Urgency.NORMAL)
note.show()
with Assistant(credentials, device_model_id) as assistant:
global device_id_global, device_model_id_global
events = assistant.start()
device_id = assistant.device_id
print('device_model_id:', device_model_id)
print('device_id:', device_id + '\n')
device_id_global = device_id
device_model_id_global = device_model_id
if should_register or (device_id != last_device_id):
if args.project_id:
register_device(args.project_id, credentials,
device_model_id, device_id, args.nickname)
pathlib.Path(os.path.dirname(args.device_config)).mkdir(
exist_ok=True)
with open(args.device_config, 'w') as f:
json.dump({
'last_device_id': device_id,
'model_id': device_model_id,
}, f)
else:
print(WARNING_NOT_REGISTERED)
for event in events:
if event.type == EventType.ON_START_FINISHED and args.query:
assistant.send_text_query(args.query)
process_event(event)
def simple_report():
"""Example of parsed pytest-json report contents."""
current_folder = pathlib.Path(__file__).resolve().parent
return json.loads((current_folder / "simple_report.json").read_text())
"en-US,en;q=0.9,de-DE;q=0.8,de;q=0.7",
'Cache-Control':
"no-cache",
'Connection':
"keep-alive",
'User-Agent':
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.80 Safari/537.36",
}
base_url = "https://www.oxfordlearnersdictionaries.com/external/pdf/wordlists/oxford-3000-5000/"
pdf_files = [("The%20Oxford%203000.pdf", "ox3k.pdf"),
("The%20Oxford%205000.pdf", "ox5k.pdf")]
for in_fn, out_fn in pdf_files:
url = base_url + in_fn
path = pl.Path(out_fn)
if path.exists():
continue
print("downloading", url)
resp = requests.get(url, headers=HEADERS)
with path.open(mode="wb") as fobj:
fobj.write(resp.content)
GRAMMAR_QUALIFIERS = {
"n": "noun",
"v": "verb",
"noun": "noun",
"adj": "adjective",
"adv": "adverb",
"prep": "preposition",
def _multinode_transfer(method, dest, source, dst, username, ssh_private_key,
rls, mpt):
# type: (str, DestinationSettings, SourceSettings, str, str,
# pathlib.Path, dict, int) -> None
"""Transfer data to multiple destination nodes simultaneously
:param str method: transfer method
:param DestinationSettings dest: destination settings
:param SourceSettings source: source settings
:param str dst: destination path
:param str username: username
:param pathlib.Path: ssh private key
:param dict rls: remote login settings
:param int mpt: max parallel transfers per node
"""
src = source.path
src_incl = source.include
src_excl = source.exclude
psrc = pathlib.Path(src)
# if source isn't a directory, convert it using src_incl
if not psrc.is_dir():
src_excl = None
src_incl = [src]
src = str(psrc.parent)
psrc = psrc.parent
# if split is specified, force to multinode_scp
if (dest.data_transfer.split_files_megabytes is not None
and method != 'multinode_scp'):
logger.warning('forcing transfer method to multinode_scp with split')
method = 'multinode_scp'
buckets = {}
files = {}
rcodes = {}
spfiles = []
spfiles_count = {}
spfiles_count_lock = threading.Lock()
for rkey in rls:
buckets[rkey] = 0
files[rkey] = []
rcodes[rkey] = None
# walk the directory structure
# 1. construct a set of dirs to create on the remote side
# 2. binpack files to different nodes
total_files = 0
dirs = set()
if dest.relative_destination_path is not None:
dirs.add(dest.relative_destination_path)
for entry in util.scantree(src):
rel = pathlib.Path(entry.path).relative_to(psrc)
sparent = str(pathlib.Path(entry.path).relative_to(psrc).parent)
if entry.is_file():
srel = str(rel)
# check filters
if src_excl is not None:
inc = not any([fnmatch.fnmatch(srel, x) for x in src_excl])
else:
inc = True
if src_incl is not None:
inc = any([fnmatch.fnmatch(srel, x) for x in src_incl])
if not inc:
logger.debug('skipping file {} due to filters'.format(
entry.path))
continue
if dest.relative_destination_path is None:
dstpath = '{}{}'.format(dst, rel)
else:
dstpath = '{}{}/{}'.format(dst, dest.relative_destination_path,
rel)
# get key of min bucket values
fsize = entry.stat().st_size
if (dest.data_transfer.split_files_megabytes is not None
and fsize > dest.data_transfer.split_files_megabytes):
nsplits = int(
math.ceil(fsize /
dest.data_transfer.split_files_megabytes))
lpad = int(math.log10(nsplits)) + 1
spfiles.append(dstpath)
spfiles_count[dstpath] = nsplits
n = 0
curr = 0
while True:
end = curr + dest.data_transfer.split_files_megabytes
if end > fsize:
end = fsize
key = min(buckets, key=buckets.get)
buckets[key] += (end - curr)
if n == 0:
dstfname = dstpath
else:
dstfname = '{}.{}{}'.format(dstpath,
_FILE_SPLIT_PREFIX,
str(n).zfill(lpad))
files[key].append((entry.path, dstfname, curr, end))
if end == fsize:
break
curr = end
n += 1
else:
key = min(buckets, key=buckets.get)
buckets[key] += fsize
files[key].append((entry.path, dstpath, None, None))
total_files += 1
# add directory to create
if sparent != '.':
if dest.relative_destination_path is None:
dirs.add(sparent)
else:
dirs.add('{}/{}'.format(dest.relative_destination_path,
sparent))
total_size = sum(buckets.values())
if total_files == 0:
logger.error('no files to ingress')
return
# create remote directories via ssh
if len(dirs) == 0:
logger.debug('no remote directories to create')
else:
logger.debug('creating remote directories: {}'.format(dirs))
dirs = ['mkdir -p {}'.format(x) for x in list(dirs)]
dirs.insert(0, 'cd {}'.format(dst))
_rls = next(iter(rls.values()))
ip = _rls.remote_login_ip_address
port = _rls.remote_login_port
del _rls
mkdircmd = ('ssh -T -x -o StrictHostKeyChecking=no '
'-o UserKnownHostsFile={} -i {} -p {} {}@{} {}'.format(
os.devnull, ssh_private_key, port, username, ip,
util.wrap_commands_in_shell(dirs)))
rc = util.subprocess_with_output(mkdircmd,
shell=True,
suppress_output=True)
if rc == 0:
logger.info('remote directories created on {}'.format(dst))
else:
logger.error('remote directory creation failed')
return
del ip
del port
logger.info(
'ingress data: {0:.4f} MiB in {1} files to transfer, using {2} max '
'parallel transfers per node'.format(total_size / _MEGABYTE,
total_files, mpt))
logger.info('begin ingressing data from {} to {}'.format(src, dst))
nodekeys = list(buckets.keys())
threads = []
start = datetime.datetime.now()
for i in range(0, len(buckets)):
nkey = nodekeys[i]
thr = threading.Thread(
target=_multinode_thread_worker,
args=(method, mpt, nkey, rcodes, files[nkey], spfiles_count,
spfiles_count_lock, rls[nkey].remote_login_ip_address,
rls[nkey].remote_login_port, username, ssh_private_key,
dest.data_transfer.scp_ssh_extra_options,
dest.data_transfer.rsync_extra_options))
threads.append(thr)
thr.start()
for i in range(0, len(buckets)):
threads[i].join()
diff = datetime.datetime.now() - start
del threads
success = True
for nkey in rcodes:
if rcodes[nkey] != 0:
logger.error('data ingress failed to node: {}'.format(nkey))
success = False
if success:
logger.info(
'finished ingressing {0:.4f} MB of data in {1} files from {2} to '
'{3} in {4:.2f} sec ({5:.3f} Mbit/s)'.format(
total_size / _MEGABYTE, total_files, src, dst,
diff.total_seconds(),
(total_size * 8 / 1e6) / diff.total_seconds()))
def ingress_data(batch_client,
compute_client,
network_client,
config,
rls=None,
kind=None,
total_vm_count=None,
to_fs=None):
# type: (batch.BatchServiceClient,
# azure.mgmt.compute.ComputeManagementClient, dict, dict, str,
# int, str) -> list
"""Ingresses data into Azure
:param batch_client: The batch client to use.
:type batch_client: `batchserviceclient.BatchServiceClient`
:param azure.mgmt.compute.ComputeManagementClient compute_client:
compute client
:param azure.mgmt.network.NetworkManagementClient network_client:
network client
:param dict config: configuration dict
:param dict rls: remote login settings
:param str kind: 'all', 'shared', 'storage', or 'remotefs'
:param int total_vm_count: total current vm count
:param str to_fs: to remote filesystem
:rtype: list
:return: list of storage threads
"""
storage_threads = []
files = settings.global_resources_files(config)
if util.is_none_or_empty(files):
logger.info('no files to ingress detected')
return storage_threads
pool = settings.pool_settings(config)
is_windows = settings.is_windows_pool(config)
for fdict in files:
source = settings.files_source_settings(fdict)
dest = settings.files_destination_settings(fdict)
if (dest.shared_data_volume is not None
and dest.storage_account_settings is not None):
raise RuntimeError(
'cannot specify both shared data volume and storage for the '
'destination for source: {}'.format(source.path))
direct_single_node = False
if dest.relative_destination_path is not None:
if dest.storage_account_settings is not None:
raise RuntimeError(
'cannot specify a relative destination path for ingress '
'to storage; use the --collate option in blobxfer '
'instead.')
# check if this is going to a single vm
if dest.shared_data_volume is None:
if total_vm_count == 1:
direct_single_node = True
elif kind == 'storage':
# this is to prevent total_vm_count check below for
# non shared/all targets and will force continuation
# of the loop below
direct_single_node = True
elif total_vm_count is None:
raise ValueError('total_vm_count is not set')
else:
raise RuntimeError(
'Cannot ingress data directly into compute node '
'host for pools with more than one node. Please use '
'a shared data volume as the ingress destination '
'instead.')
if dest.shared_data_volume is not None or direct_single_node:
if kind == 'storage':
logger.warning(
'skipping data ingress from {} to {} for pool as ingress '
'to shared file system not specified'.format(
source.path, dest.shared_data_volume))
continue
if is_windows:
logger.error(
('cannot data ingress from {} to pool {} with windows '
'compute nodes').format(source.path, pool.id))
continue
# get rfs settings
rfs = None
dst_rfs = False
# set base dst path
dst = '{}/batch/tasks/mounts'.format(
settings.temp_disk_mountpoint(config))
# convert shared to actual path
if not direct_single_node:
sdv = settings.global_resources_shared_data_volumes(config)
for sdvkey in sdv:
if sdvkey == dest.shared_data_volume:
if settings.is_shared_data_volume_gluster_on_compute(
sdv, sdvkey):
if kind == 'remotefs':
continue
dst = '{}/{}/'.format(
dst, settings.get_gluster_on_compute_volume())
elif settings.is_shared_data_volume_storage_cluster(
sdv, sdvkey):
if kind != 'remotefs' or sdvkey != to_fs:
continue
if rfs is None:
rfs = settings.remotefs_settings(config, to_fs)
dst = rfs.storage_cluster.file_server.mountpoint
# add trailing directory separator if needed
if dst[-1] != '/':
dst = dst + '/'
dst_rfs = True
else:
raise RuntimeError(
'data ingress to {} not supported'.format(
sdvkey))
break
# skip entries that are a mismatch if remotefs transfer
# is selected
if kind == 'remotefs':
if not dst_rfs:
continue
else:
if dst_rfs:
continue
# set ssh info
if dst_rfs:
username = rfs.storage_cluster.ssh.username
# retrieve public ips from all vms in named storage cluster
rls = {}
for i in range(rfs.storage_cluster.vm_count):
vm_name = '{}-vm{}'.format(
rfs.storage_cluster.hostname_prefix, i)
vm = compute_client.virtual_machines.get(
resource_group_name=rfs.storage_cluster.resource_group,
vm_name=vm_name,
)
_, pip = resource.get_nic_and_pip_from_virtual_machine(
network_client, rfs.storage_cluster.resource_group, vm)
# create compute node rls settings with sc vm ip/port
rls[vm_name] = \
batchmodels.ComputeNodeGetRemoteLoginSettingsResult(
remote_login_ip_address=pip.ip_address,
remote_login_port=22)
else:
username = pool.ssh.username
if rls is None:
logger.warning(
'skipping data ingress from {} to {} for pool with no '
'remote login settings or non-existent pool'.format(
source.path, dest.shared_data_volume))
continue
if username is None:
raise RuntimeError(
'cannot ingress data to shared data volume without a '
'valid SSH user')
# try to get valid ssh private key (from various config blocks)
ssh_private_key = dest.data_transfer.ssh_private_key
if ssh_private_key is None:
ssh_private_key = pool.ssh.ssh_private_key
if ssh_private_key is None:
ssh_private_key = pathlib.Path(crypto.get_ssh_key_prefix())
if not ssh_private_key.exists():
raise RuntimeError(
'specified SSH private key is invalid or does not '
'exist')
logger.debug(
'using ssh_private_key from: {}'.format(ssh_private_key))
if (dest.data_transfer.method == 'scp'
or dest.data_transfer.method == 'rsync+ssh'):
# split/source include/exclude will force multinode
# transfer with mpt=1
if (dest.data_transfer.split_files_megabytes is not None
or source.include is not None
or source.exclude is not None):
_multinode_transfer(
'multinode_' + dest.data_transfer.method, dest, source,
dst, username, ssh_private_key, rls, 1)
else:
_singlenode_transfer(dest, source.path, dst, username,
ssh_private_key, rls)
elif (dest.data_transfer.method == 'multinode_scp'
or dest.data_transfer.method == 'multinode_rsync+ssh'):
_multinode_transfer(
dest.data_transfer.method, dest, source, dst, username,
ssh_private_key, rls,
dest.data_transfer.max_parallel_transfers_per_node)
else:
raise RuntimeError('unknown transfer method: {}'.format(
dest.data_transfer.method))
elif dest.storage_account_settings is not None:
if kind == 'shared':
logger.warning(
'skipping data ingress from {} for pool as ingress '
'to Azure Blob/File Storage not specified'.format(
source.path))
continue
thr = _azure_blob_storage_transfer(
settings.credentials_storage(config,
dest.storage_account_settings),
dest.data_transfer, source)
storage_threads.append(thr)
else:
raise RuntimeError(
'invalid file transfer configuration: {}'.format(fdict))
return storage_threads
def _singlenode_transfer(dest, src, dst, username, ssh_private_key, rls):
# type: (DestinationSettings, str, str, pathlib.Path, dict) -> None
"""Transfer data to a single node
:param DestinationSettings dest: destination settings
:param str src: source path
:param str dst: destination path
:param str username: username
:param pathlib.Path: ssh private key
:param dict rls: remote login settings
"""
# get remote settings
_rls = next(iter(rls.values()))
ip = _rls.remote_login_ip_address
port = _rls.remote_login_port
del _rls
# modify dst with relative dest
if util.is_not_empty(dest.relative_destination_path):
dst = '{}{}'.format(dst, dest.relative_destination_path)
# create relative path on host
logger.debug('creating remote directory: {}'.format(dst))
dirs = ['mkdir -p {}'.format(dst)]
mkdircmd = ('ssh -T -x -o StrictHostKeyChecking=no '
'-o UserKnownHostsFile={} -i {} -p {} {}@{} {}'.format(
os.devnull, ssh_private_key, port, username, ip,
util.wrap_commands_in_shell(dirs)))
rc = util.subprocess_with_output(mkdircmd,
shell=True,
suppress_output=True)
if rc == 0:
logger.info('remote directories created on {}'.format(dst))
else:
logger.error('remote directory creation failed')
return
del dirs
# determine if recursive flag must be set
psrc = pathlib.Path(src)
recursive = '-r' if psrc.is_dir() else ''
# set command source path and adjust dst path
if recursive:
cmdsrc = '.'
else:
cmdsrc = shellquote(src)
# transfer data
if dest.data_transfer.method == 'scp':
cmd = ('scp -o StrictHostKeyChecking=no '
'-o UserKnownHostsFile={} -p {} {} -i {} '
'-P {} {} {}@{}:"{}"'.format(
os.devnull,
dest.data_transfer.scp_ssh_extra_options, recursive,
ssh_private_key.resolve(), port, cmdsrc, username, ip,
shellquote(dst)))
elif dest.data_transfer.method == 'rsync+ssh':
cmd = ('rsync {} {} -e "ssh -T -x -o StrictHostKeyChecking=no '
'-o UserKnownHostsFile={} {} -i {} -p {}" {} {}@{}:"{}"'.format(
dest.data_transfer.rsync_extra_options, recursive,
os.devnull, dest.data_transfer.scp_ssh_extra_options,
ssh_private_key.resolve(), port, cmdsrc, username, ip,
shellquote(dst)))
else:
raise ValueError('Unknown transfer method: {}'.format(
dest.data_transfer.method))
logger.info('begin ingressing data from {} to {}'.format(src, dst))
start = datetime.datetime.now()
rc = util.subprocess_with_output(cmd,
shell=True,
cwd=src if recursive else None)
diff = datetime.datetime.now() - start
if rc == 0:
logger.info(
'finished ingressing data from {0} to {1} in {2:.2f} sec'.format(
src, dst, diff.total_seconds()))
else:
logger.error(
'data ingress from {} to {} failed with return code: {}'.format(
src, dst, rc))
import pathlib2 as pathlib
import json
my_file = open(pathlib.Path(__file__).parent / 'data.json', 'r')
file_contents = json.load(my_file) #Lo convierte en diccionario
my_file.close()
print(file_contents['friends'][0])
#Guardando en archivo .json
cars = [{
"made": "ford",
"model": "Fiesta"
}, {
"made": "ford",
"model": "Focus"
}]
with open(pathlib.Path(__file__).parent / 'cars.json', 'w',
encoding='utf-8') as f:
json.dump(cars, f)
def _plot_dataset(ds, net_code, ref_code):
# Sort ds rows by SNR, so that the weakest SNR points are drawn first and the high SNR point last,
# to make sure high SNR point are in the top rendering layer.
ds = ds.sort_values('snr')
times = pandas_timestamp_to_plottable_datetime(ds['originTimestamp'])
vals = ds[yaxis].values
qual = ds['snr'].values
min_mag = 4.0
mag = ds['mag'].values - min_mag
ylabel = 'Relative TT residual (sec)'
title = r"Station {} TT residuals relative to network {} (filtering: ref SNR$\geq${}, CWT$\geq${}, "\
r"slope$\geq${}, $n\sigma\geq{}$)".format(ref_code, net_code, str(filter_options.min_event_snr),
str(filter_options.cwt_cutoff),
str(filter_options.slope_cutoff),
str(filter_options.nsigma_cutoff))
if vals.any():
plt.figure(figsize=(32, 9))
sc = plt.scatter(times,
vals,
c=qual,
alpha=0.5,
cmap='gnuplot_r',
s=np.maximum(50 * mag, 10),
edgecolors=None,
linewidths=0)
time_formatter = matplotlib.dates.DateFormatter("%Y-%m-%d")
plt.axes().xaxis.set_major_formatter(time_formatter)
cb = plt.colorbar(sc, drawedges=False)
cb.set_label('Signal to noise ratio', fontsize=12)
plt.grid(color='#808080', linestyle=':', alpha=0.7)
plt.xlabel(xlabel, fontsize=14)
plt.ylabel(ylabel, fontsize=14)
plt.xticks(fontsize=14)
plt.yticks(fontsize=14)
plt.xlim(time_range)
plt.ylim((-tt_scale, tt_scale))
plt.clim(snr_scale)
plt.title(title, fontsize=18)
plt.legend(['Point size = Mag - {}, Color = SNR'.format(min_mag)],
fontsize=12,
loc=1)
plt.text(0.01,
0.96,
"Channel selection: {}".format(
filter_options.channel_preference),
transform=plt.gca().transAxes,
fontsize=12)
plt.text(0.01,
0.92,
"Start date: {}".format(str(time_range[0])),
transform=plt.gca().transAxes,
fontsize=12)
plt.text(0.01,
0.88,
" End date: {}".format(str(time_range[1])),
transform=plt.gca().transAxes,
fontsize=12)
plt.tight_layout(pad=1.05)
if annotator is not None:
annotator()
if save_file:
subfolder = os.path.join(
ref_code.split('.')[0] + file_label, net_code)
# subfolder = net_code
pathlib.Path(subfolder).mkdir(exist_ok=True, parents=True)
plt_file = os.path.join(
subfolder, '_'.join([ref_code, net_code]) + '_' +
ylabel.replace(" ", "").replace(".*", "") + ".png")
plt.savefig(plt_file, dpi=150)
plt.close()
else: # pragma: no cover
plt.show()
def test_descriptor_read_data(tmpdir):
tmpdir.join('a').write('ab')
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
# test normal
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._resume = mock.MagicMock()
ud._resume.return_value = None
# test no data to read
mockoffsets = mock.MagicMock()
mockoffsets.num_bytes = 0
data, newoffset = ud.read_data(mockoffsets)
assert data is None
assert newoffset is None
# test normal data to read
offsets, rb = ud.next_offsets()
assert rb is None
data, newoffset = ud.read_data(offsets)
assert data == b'a'
assert newoffset is None
# test stdin
with mock.patch('blobxfer.STDIN',
new_callable=mock.PropertyMock) as patched_stdin:
patched_stdin.read = mock.MagicMock()
patched_stdin.read.return_value = b'z'
ud.local_path.use_stdin = True
data, newoffset = ud.read_data(offsets)
assert data == b'z'
assert newoffset.chunk_num == 0
assert newoffset.num_bytes == 1
assert newoffset.range_start == 0
assert newoffset.range_end == 0
assert not newoffset.pad
assert ud._total_chunks == 3
assert ud._outstanding_ops == 3
assert ud._offset == 1
assert ud.entity.size == 2
with mock.patch('blobxfer.STDIN',
new_callable=mock.PropertyMock) as patched_stdin:
patched_stdin.read = mock.MagicMock()
patched_stdin.read.return_value = None
ud.local_path.use_stdin = True
data, newoffset = ud.read_data(offsets)
assert data is None
assert newoffset is None
assert ud._total_chunks == 2
assert ud._outstanding_ops == 2
assert ud._chunk_num == 0
# 'I1_min' : 1.0,
# 'I1_max' : 1.0,
# 'I2_min' : 1.0,
# 'I2_max' : 8.0
# }
I1_space = np.arange(constraints['I1_min'], constraints['I1_max'] + step, step)
I1_number = int((constraints['I1_max'] + step - constraints['I1_min']) / step)
I2_space = np.arange(constraints['I2_min'], constraints['I2_max'] + step, step)
I2_number = int((constraints['I2_max'] + step - constraints['I2_min']) / step)
grid = np.zeros((I1_number, I2_number))
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M")
final_data_saving_folder = data_saving_folder + '\extracted_features' + '_' + str(timestamp) + '\\'
pathlib.Path(final_data_saving_folder).mkdir(parents = True, exist_ok = True)
maps_saving_folder = final_data_saving_folder + '\maps' + '\\'
pathlib.Path(maps_saving_folder).mkdir(parents = True, exist_ok = True)
feature_file_path = final_data_saving_folder + 'features' + '.txt' # + '_' + str(timestamp) + ".txt"
feature_file = open(feature_file_path,"w+")
failures_file_path = final_data_saving_folder + 'failures' + ".txt"
failure_file = open(failures_file_path,"w+")
def log_failure(I1, I2, message = ""):
error_message = '%.2f' % I1 + ' ' + '%.2f' % I2 + ' ' + message + '\n'
print(error_message)
failure_file.write(error_message)
# HERE YOU PUT THE FUNCTIONS THAT WILL EXTRACT THE FINAL FEATURES YOU NEED
# pointer to the function : array of args it needs
# feature_functions_list = [
def get_file_entry(self):
local_file = None
# don't provide file in case this event is out of the history window
last_count = self._get_metric_count(self.metric,
self.variant,
next=False)
if abs(self._count - last_count) > self._file_history_size:
output = None
elif isinstance(self._image_data, (six.StringIO, six.BytesIO)):
output = self._image_data
elif self._image_data is not None:
image_data = self._image_data
if not isinstance(image_data, np.ndarray):
# try conversion, if it fails we'll leave it to the user.
image_data = np.ndarray(image_data, dtype=np.uint8)
image_data = np.atleast_3d(image_data)
if image_data.dtype != np.uint8:
if np.issubdtype(image_data.dtype,
np.floating) and image_data.max() <= 1.0:
image_data = (image_data * 255).astype(np.uint8)
else:
image_data = image_data.astype(np.uint8)
shape = image_data.shape
height, width, channel = shape[:3]
if channel == 1:
image_data = np.reshape(image_data, (height, width))
# serialize image
image = Image.fromarray(image_data)
output = six.BytesIO()
image_format = Image.registered_extensions().get(
self._format.lower(), 'JPEG')
image.save(output, format=image_format, quality=self._quality)
output.seek(0)
else:
# noinspection PyBroadException
try:
output = pathlib2.Path(self._local_image_path)
if not output.is_file():
output = None
except Exception:
output = None
if output is None:
from ...debugging.log import LoggerRoot
LoggerRoot.get_base_logger().warning(
'Skipping upload, could not find object file \'{}\''.
format(self._local_image_path))
return None
return self.FileEntry(
event=self,
name=self._upload_filename,
stream=output,
url_prop='url',
key_prop='key',
upload_uri=self._upload_uri,
delete_local_file=local_file
if self._delete_after_upload else None,
retries=self.retries,
)
def test_descriptor_generate_metadata(tmpdir):
tmpdir.join('a').write('ab')
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
# test nothing
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.attributes = False
opts.store_file_properties.md5 = False
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
meta = ud.generate_metadata()
assert meta is None
# test page md5 align
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.attributes = False
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Page
ase._name = 'name'
ase._encryption = None
ase._size = 1
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._offset = 1
ud.md5 = hashlib.md5()
ud.md5.update(b'z')
meta = ud.generate_metadata()
assert meta is None
md5 = hashlib.md5()
md5.update(b'z' + b'\0' * 511)
assert ud.md5.digest() == md5.digest()
# test fileattr meta
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.attributes = True
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
# file attr store is not avail on windows
if not util.on_windows():
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
meta = ud.generate_metadata()
assert metadata.JSON_KEY_BLOBXFER_METADATA in meta
assert metadata._JSON_KEY_FILE_ATTRIBUTES in meta[
metadata.JSON_KEY_BLOBXFER_METADATA]
# test enc meta
opts.store_file_properties.attributes = False
opts.store_file_properties.md5 = False
opts.rsa_public_key = 'abc'
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ase.encryption_metadata = mock.MagicMock()
ase.encryption_metadata.convert_to_json_with_mac.return_value = {
'encmeta': 'encmeta'
}
meta = ud.generate_metadata()
assert 'encmeta' in meta
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud.hmac = None
ase.encryption_metadata = mock.MagicMock()
ase.encryption_metadata.convert_to_json_with_mac.return_value = {
'encmeta': 'encmeta'
}
meta = ud.generate_metadata()
assert 'encmeta' in meta
opts.store_file_properties.md5 = True
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ase.encryption_metadata = mock.MagicMock()
ase.encryption_metadata.convert_to_json_with_mac.return_value = {
'encmeta': 'encmeta'
}
meta = ud.generate_metadata()
assert 'encmeta' in meta
# test vio meta
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
lp.view = mock.MagicMock()
lp.view.mode = upload.VectoredIoDistributionMode.Stripe
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
with mock.patch(
'blobxfer.models.metadata.generate_vectored_io_stripe_metadata',
return_value={'viometa': 'viometa'}):
meta = ud.generate_metadata()
assert metadata.JSON_KEY_BLOBXFER_METADATA in meta
assert 'viometa' in meta[metadata.JSON_KEY_BLOBXFER_METADATA]
def main(api_endpoint, credentials, project_id,
device_model_id, device_id, device_config,
lang, display, verbose,
input_audio_file, output_audio_file,
audio_sample_rate, audio_sample_width,
audio_iter_size, audio_block_size, audio_flush_size,
grpc_deadline, once, *args, **kwargs):
"""Samples for the Google Assistant API.
Examples:
Run the sample with microphone input and speaker output:
$ python -m googlesamples.assistant
Run the sample with file input and speaker output:
$ python -m googlesamples.assistant -i <input file>
Run the sample with file input and output:
$ python -m googlesamples.assistant -i <input file> -o <output file>
"""
if gender=='Male':
subprocess.Popen(["aplay", "{}/sample-audio-files/Startup-Male.wav".format(ROOT_PATH)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
else:
subprocess.Popen(["aplay", "{}/sample-audio-files/Startup-Female.wav".format(ROOT_PATH)], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# Setup logging.
logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO)
# Load OAuth 2.0 credentials.
try:
with open(credentials, 'r') as f:
credentials = google.oauth2.credentials.Credentials(token=None,
**json.load(f))
http_request = google.auth.transport.requests.Request()
credentials.refresh(http_request)
except Exception as e:
logging.error('Error loading credentials: %s', e)
logging.error('Run google-oauthlib-tool to initialize '
'new OAuth 2.0 credentials.')
sys.exit(-1)
# Create an authorized gRPC channel.
grpc_channel = google.auth.transport.grpc.secure_authorized_channel(
credentials, http_request, api_endpoint)
logging.info('Connecting to %s', api_endpoint)
# Configure audio source and sink.
audio_device = None
if input_audio_file:
audio_source = audio_helpers.WaveSource(
open(input_audio_file, 'rb'),
sample_rate=audio_sample_rate,
sample_width=audio_sample_width
)
else:
audio_source = audio_device = (
audio_device or audio_helpers.SoundDeviceStream(
sample_rate=audio_sample_rate,
sample_width=audio_sample_width,
block_size=audio_block_size,
flush_size=audio_flush_size
)
)
if output_audio_file:
audio_sink = audio_helpers.WaveSink(
open(output_audio_file, 'wb'),
sample_rate=audio_sample_rate,
sample_width=audio_sample_width
)
else:
audio_sink = audio_device = (
audio_device or audio_helpers.SoundDeviceStream(
sample_rate=audio_sample_rate,
sample_width=audio_sample_width,
block_size=audio_block_size,
flush_size=audio_flush_size
)
)
# Create conversation stream with the given audio source and sink.
conversation_stream = audio_helpers.ConversationStream(
source=audio_source,
sink=audio_sink,
iter_size=audio_iter_size,
sample_width=audio_sample_width,
)
if not device_id or not device_model_id:
try:
with open(device_config) as f:
device = json.load(f)
device_id = device['id']
device_model_id = device['model_id']
logging.info("Using device model %s and device id %s",
device_model_id,
device_id)
except Exception as e:
logging.warning('Device config not found: %s' % e)
logging.info('Registering device')
if not device_model_id:
logging.error('Option --device-model-id required '
'when registering a device instance.')
sys.exit(-1)
if not project_id:
logging.error('Option --project-id required '
'when registering a device instance.')
sys.exit(-1)
device_base_url = (
'https://%s/v1alpha2/projects/%s/devices' % (api_endpoint,
project_id)
)
device_id = str(uuid.uuid1())
payload = {
'id': device_id,
'model_id': device_model_id,
'client_type': 'SDK_SERVICE'
}
session = google.auth.transport.requests.AuthorizedSession(
credentials
)
r = session.post(device_base_url, data=json.dumps(payload))
if r.status_code != 200:
logging.error('Failed to register device: %s', r.text)
sys.exit(-1)
logging.info('Device registered: %s', device_id)
pathlib.Path(os.path.dirname(device_config)).mkdir(exist_ok=True)
with open(device_config, 'w') as f:
json.dump(payload, f)
device_handler = device_helpers.DeviceRequestHandler(device_id)
@device_handler.command('action.devices.commands.OnOff')
def onoff(on):
if on:
logging.info('Turning device on')
else:
logging.info('Turning device off')
@device_handler.command('com.example.commands.BlinkLight')
def blink(speed, number):
logging.info('Blinking device %s times.' % number)
delay = 1
if speed == "slowly":
delay = 2
elif speed == "quickly":
delay = 0.5
for i in range(int(number)):
logging.info('Device is blinking.')
time.sleep(delay)
with SampleAssistant(lang, device_model_id, device_id,
conversation_stream, display,
grpc_channel, grpc_deadline,
device_handler) as assistant:
# If file arguments are supplied:
# exit after the first turn of the conversation.
if input_audio_file or output_audio_file:
assistant.assist()
return
def detected():
continue_conversation=assistant.assist()
if continue_conversation:
print('Continuing conversation')
assistant.assist()
signal.signal(signal.SIGINT, signal_handler)
sensitivity = [0.5]*len(models)
callbacks = [detected]*len(models)
detector = snowboydecoder.HotwordDetector(models, sensitivity=sensitivity)
def start_detector():
detector.start(detected_callback=callbacks,
interrupt_check=interrupt_callback,
sleep_time=0.03)
# If no file arguments supplied:
# keep recording voice requests using the microphone
# and playing back assistant response using the speaker.
# When the once flag is set, don't wait for a trigger. Otherwise, wait.
wait_for_user_trigger = not once
while True:
if wait_for_user_trigger:
if custom_wakeword:
start_detector()
else:
button_state=GPIO.input(pushbuttontrigger)
if button_state==True:
continue
else:
pass
continue_conversation = assistant.assist()
# wait for user trigger if there is no follow-up turn in
# the conversation.
wait_for_user_trigger = not continue_conversation
# If we only want one conversation, break.
if once and (not continue_conversation):
break
detector.terminate()
def test_descriptor_compute_remote_size(tmpdir):
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
# encrypted remote size with replica
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = 'abc'
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = mock.MagicMock()
ase._encryption.symmetric_key = 'abc'
ase2 = azmodels.StorageEntity('cont')
ase2._mode = azmodels.StorageModes.Block
ase2._name = 'name2'
ase.replica_targets = [ase2]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == 48
for rt in ase.replica_targets:
assert rt.size == ud.entity.size
# remote size
opts = mock.MagicMock()
opts.chunk_size_bytes = 16
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == 32
# remote size of zero
tmpdir.join('b').ensure(file=True)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('b'))
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == 0
# stdin as page, resize
lp = upload.LocalPath(pathlib.Path('-'), pathlib.Path('-'), use_stdin=True)
opts.stdin_as_page_blob_size = 0
ase._mode = azmodels.StorageModes.Page
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == upload._MAX_PAGE_BLOB_SIZE
assert ud._needs_resize
# stdin as page, no resize
lp = upload.LocalPath(pathlib.Path('-'), pathlib.Path('-'), use_stdin=True)
opts.stdin_as_page_blob_size = 32
ase._mode = azmodels.StorageModes.Page
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._compute_remote_size(opts)
assert ud.entity.size == 32
assert not ud._needs_resize
def moveFiles(new_root,barcode_dict,portal_dict,unwanted,noPortalPath,badBarcodePath,barcodeLen,dPath):
# Make all barcodes into caps for comparison
barcode_Dict=dict((k.upper(), v) for k, v in barcode_dict.items())
portal_Dict=dict((k.upper(), v) for k, v in portal_dict.items())
# Start dictionary to put new file paths in
# filename: [barcode, portal, date, current(new) path]
new_dict={}
# filename:newpath(where we want a large image but there is none)
nolarge_dict={}
# filename:[barcode,date,old_path]
badbarcode_dict={}
# filename:[barcode,date,old_path]
duplicate_dict={}
# Iterate through every barcode in image barcode dict
for b in barcode_Dict:
# if barcode is the right lenght, go through a lot of stuff.
if len(b) == int(barcodeLen):
# Split apart letters and numbers from barcode
try:
b_letters,b_numbers = ["".join(x) for _, x in itertools.groupby(b, key=str.isdigit)]
# For bad barcodes move them to a special folder for Jennie to check.
except ValueError:
for p in barcode_Dict[b]:
# Add files to badbarcode dict, get creation date, and move to their special place.
badbarcode_dict=badBarcodeSequence(p,b,barcode_Dict,unwanted,badBarcodePath,badbarcode_dict)
# Loop goes here after try if there are no exceptions
# For all good barcodes that can be split into Letters/Numbers
else:
#print(b+", good barcode")
# If barcode is found in records, move it into correct portal file
if b in portal_Dict:
#print(b+", in portal dict")
# Get portal for barcode
portal=portal_Dict[b]
# Iterate through all file paths in barcode dict
for p in barcode_Dict[b]:
if os.path.exists(p):
# Ignore files in "unwanted" list
if any(x in p for x in unwanted):
pass
else:
#print(p+", good path")
# Get new file path and uppercase file name
newDir,newPath,fileName=newPathNames(b,p,new_root,portal)
# Check if file exists at new path
if not os.path.exists(newPath):
#print(newPath+", new path")
# Make new directories if needed https://docs.python.org/3/library/pathlib.html
if not os.path.exists(newDir):
pathlib.Path(newDir).mkdir(parents=True, exist_ok=True)
# Get creation date
d = creation_date(p)
#filename: [barcode, portal, date, current(new) path]
new_dict[fileName]=[b,portal,d,newPath]
# Copy file, preserving permissions
print(newPath+", good barcode, in portal list")
shutil.move(p,newPath)
# If path exists, check if this is a rerun, if not, put newest file in folder. make note of duplicates
elif os.path.exists(newPath):
print(newPath+", duplicate, in portal dict")
# Get creation dates for file already moved, and the one that is similar to it. likely different due to case sensitive issues.
# This got really ugly when moving files and dealing with files already moved getting passed to this loop.
# Not making it clean right now. don't care enough. just need to move files
try:
d = creation_date(p)
except FileNotFoundError:
d = creation_date(newPath)
d1 = creation_date(newPath)
# Copy both duplicate files to new folder. add _1 to one of them.
dName=os.path.basename(p)
ddPath=os.path.join(dPath,dName.upper())
try:
pass
#shutil.copy2(p,ddPath)
except FileNotFoundError:
pass
d1Name=os.path.basename(newPath)
dd1Path=os.path.join(dPath,d1Name.upper(),'1')
try:
pass
#shutil.copy2(p,dd1Path)
except FileNotFoundError:
pass
# If dates are the same, probably rerunning script, dont add to duplicate dict
if d == d1:
new_dict[fileName]=[b,portal,d,newPath]
#pass
else:
# Add to duplicate dict
duplicate_dict[fileName]=[b,d,p]
# If this file is newer, replace older file. Rerun or not we want to move newer file to main folder.
if d > d1:
#print(p+", replace older image file ")
#filename: [barcode, portal, date, current(new) path]
new_dict[fileName]=[b,portal,d,newPath]
# Copy file, preserving permissions
#print(ddPath+", duplicate path")
shutil.move(p,newPath)
# If the path to the original image doesnt exist. In the case of having to rerun this script halfway through.
else:
# File is already moved add to moved dict.
newDir,newPath,fileName=newPathNames(b,p,new_root,portal)
d = creation_date(newPath)
new_dict[fileName]=[b,portal,d,newPath]
# If no record in master list. Move to special folder.
elif b not in portal_Dict:
#print(b+", not in records")
# Iterate through all file paths in barcode dict
for p in barcode_Dict[b]:
# Ignore files in "unwanted" list
if any(x in p for x in unwanted):
pass
else:
# Make new path to folder for images that don't have barcode in master list.
fName=os.path.basename(p)
newPath=os.path.join(noPortalPath,fName.upper())
# Check if file exists at new path
if not os.path.exists(newPath):
# Get creation date
d = creation_date(p)
#filename: [barcode, portal, date, current(new) path]
new_dict[fileName]=[b,"NoPortal",d,newPath]
# Copy file, preserving permissions
print(newPath+", not in portal dict")
shutil.move(p,newPath)
# If path exists, check if this is a rerun, if not, put newest file in folder. make note of duplicates
elif os.path.exists(newPath):
print(newPath+", duplicate, not in portal dict")
# Get creation dates for file already moved, and the one that is similar to it.
# likely different due to case sensitive issues.
try:
d = creation_date(p)
except FileNotFoundError:
d = creation_date(newPath)
d1 = creation_date(newPath)
# Copy both duplicate files to new folder. add _1 to one of them.
dName=os.path.basename(p)
ddPath=os.path.join(dPath,dName.upper())
try:
pass
#shutil.copy2(p,ddPath)
except FileNotFoundError:
pass
d1Name=os.path.basename(newPath)
dd1Path=os.path.join(dPath,d1Name.upper(),'1')
try:
pass
#shutil.copy2(p,dd1Path)
except FileNotFoundError:
pass
# If dates are the same, probably rerunning script, dont add to duplicate dict
if d == d1:
new_dict[fName.upper()]=[b,"NoPortal",d,newPath]
#pass
else:
# Add to duplicate dict
duplicate_dict[fileName]=[b,d,p]
# If this file is newer, replace older file. Rerun or not we want to move newer file to main folder.
if d > d1:
# Copy file, preserving permissions
shutil.move(p,newPath)
#filename: [barcode, portal, date, current(new) path]
new_dict[fName.upper()]=[b,"NoPortal",d,newPath]
#print("No record for "+str(b)+" moved to "+str(noPortalPath))
# If barcode is wrong lenght, shove it somewhere else, and make note.
else:
for p in barcode_Dict[b]:
badbarcode_dict=badBarcodeSequence(p,b,barcode_Dict,unwanted,badBarcodePath,badbarcode_dict)
return new_dict,badbarcode_dict,duplicate_dict
def test_descriptor_adjust_chunk_size(tmpdir):
tmpdir.join('a').ensure(file=True)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 0
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 0
with mock.patch('blobxfer.models.upload._DEFAULT_AUTO_CHUNKSIZE_BYTES', 1):
with mock.patch(
'blobxfer.models.upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES', 3):
with mock.patch('blobxfer.models.upload._MAX_NUM_CHUNKS', 2):
tmpdir.join('a').write('z' * 4)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)),
pathlib.Path('a'))
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 2
lp = upload.LocalPath(pathlib.Path(str(tmpdir)),
pathlib.Path('-'),
use_stdin=True)
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES
tmpdir.join('a').write('z' * 32)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Page
ase._name = 'name'
ase._encryption = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 32
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Append
ase._name = 'name'
ase._encryption = None
opts.chunk_size_bytes = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES + 1
with mock.patch(
'blobxfer.models.upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES', 4):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 4
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
opts.chunk_size_bytes = 32
opts.one_shot_bytes = 32
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 32
opts.one_shot_bytes = 31
with mock.patch('blobxfer.models.upload._MAX_BLOCK_BLOB_CHUNKSIZE_BYTES',
4):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 4
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.File
ase._name = 'name'
ase._encryption = None
opts.chunk_size_bytes = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES + 1
with mock.patch(
'blobxfer.models.upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES', 4):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 4
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Page
ase._name = 'name'
ase._encryption = None
opts.chunk_size_bytes = upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES + 1
with mock.patch(
'blobxfer.models.upload._MAX_NONBLOCK_BLOB_CHUNKSIZE_BYTES', 4):
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
assert ud._chunk_size == 4
with mock.patch('blobxfer.models.upload._MAX_PAGE_BLOB_SIZE', 4):
with pytest.raises(RuntimeError):
upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
def main(api_endpoint, credentials, project_id,
device_model_id, device_id, device_config, lang, verbose,
input_audio_file, output_audio_file,
audio_sample_rate, audio_sample_width,
audio_iter_size, audio_block_size, audio_flush_size,
grpc_deadline, once, *args, **kwargs):
"""Samples for the Google Assistant API.
Examples:
Run the sample with microphone input and speaker output:
$ python -m googlesamples.assistant
Run the sample with file input and speaker output:
$ python -m googlesamples.assistant -i <input file>
Run the sample with file input and output:
$ python -m googlesamples.assistant -i <input file> -o <output file>
"""
# Setup logging.
logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO)
# Load OAuth 2.0 credentials.
try:
with open(credentials, 'r') as f:
credentials = google.oauth2.credentials.Credentials(token=None,
**json.load(f))
http_request = google.auth.transport.requests.Request()
credentials.refresh(http_request)
except Exception as e:
logging.error('Error loading credentials: %s', e)
logging.error('Run google-oauthlib-tool to initialize '
'new OAuth 2.0 credentials.')
sys.exit(-1)
# Create an authorized gRPC channel.
grpc_channel = google.auth.transport.grpc.secure_authorized_channel(
credentials, http_request, api_endpoint)
logging.info('Connecting to %s', api_endpoint)
# Configure audio source and sink.
audio_device = None
if input_audio_file:
audio_source = audio_helpers.WaveSource(
open(input_audio_file, 'rb'),
sample_rate=audio_sample_rate,
sample_width=audio_sample_width
)
else:
audio_source = audio_device = (
audio_device or audio_helpers.SoundDeviceStream(
sample_rate=audio_sample_rate,
sample_width=audio_sample_width,
block_size=audio_block_size,
flush_size=audio_flush_size
)
)
if output_audio_file:
audio_sink = audio_helpers.WaveSink(
open(output_audio_file, 'wb'),
sample_rate=audio_sample_rate,
sample_width=audio_sample_width
)
else:
audio_sink = audio_device = (
audio_device or audio_helpers.SoundDeviceStream(
sample_rate=audio_sample_rate,
sample_width=audio_sample_width,
block_size=audio_block_size,
flush_size=audio_flush_size
)
)
# Create conversation stream with the given audio source and sink.
conversation_stream = audio_helpers.ConversationStream(
source=audio_source,
sink=audio_sink,
iter_size=audio_iter_size,
sample_width=audio_sample_width,
)
device_handler = device_helpers.DeviceRequestHandler(device_id)
@device_handler.command('action.devices.commands.OnOff')
def onoff(on):
if on:
logging.info('Turning device on')
else:
logging.info('Turning device off')
if not device_id or not device_model_id:
try:
with open(device_config) as f:
device = json.load(f)
device_id = device['id']
device_model_id = device['model_id']
except Exception as e:
logging.warning('Device config not found: %s' % e)
logging.info('Registering device')
if not device_model_id:
logging.error('Option --device-model-id required '
'when registering a device instance.')
sys.exit(-1)
if not project_id:
logging.error('Option --project-id required '
'when registering a device instance.')
sys.exit(-1)
device_base_url = (
'https://%s/v1alpha2/projects/%s/devices' % (api_endpoint,
project_id)
)
device_id = str(uuid.uuid1())
payload = {
'id': device_id,
'model_id': device_model_id,
'client_type': 'SDK_SERVICE'
}
session = google.auth.transport.requests.AuthorizedSession(
credentials
)
r = session.post(device_base_url, data=json.dumps(payload))
if r.status_code != 200:
logging.error('Failed to register device: %s', r.text)
sys.exit(-1)
logging.info('Device registered: %s', device_id)
pathlib.Path(os.path.dirname(device_config)).mkdir(exist_ok=True)
with open(device_config, 'w') as f:
json.dump(payload, f)
with SampleAssistant(lang, device_model_id, device_id,
conversation_stream,
grpc_channel, grpc_deadline,
device_handler) as assistant:
# If file arguments are supplied:
# exit after the first turn of the conversation.
if input_audio_file or output_audio_file:
assistant.assist()
return
# If no file arguments supplied:
# keep recording voice requests using the microphone
# and playing back assistant response using the speaker.
# When the once flag is set, don't wait for a trigger. Otherwise, wait.
wait_for_user_trigger = not once
while True:
if wait_for_user_trigger:
click.pause(info='Press Enter to send a new request...')
continue_conversation = assistant.assist()
# wait for user trigger if there is no follow-up turn in
# the conversation.
wait_for_user_trigger = not continue_conversation
# If we only want one conversation, break.
if once and (not continue_conversation):
break
def test_resume(tmpdir):
tmpdir.join('a').write('zz')
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 0
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
# test no resume
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), None)
assert ud._resume() is None
# check if path exists in resume db
resume = mock.MagicMock()
resume.get_record.return_value = None
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() is None
# check same lengths
bad = mock.MagicMock()
bad.length = 0
resume.get_record.return_value = bad
assert ud._resume() is None
# check completed resume
comp = mock.MagicMock()
comp.length = 2
comp.completed = True
comp.total_chunks = 1
comp.chunk_size = 2
comp.completed_chunks = 1
resume.get_record.return_value = comp
ud._completed_chunks = mock.MagicMock()
ud._src_ase = ase
assert ud._resume() == 2
ase.replica_targets = [ase]
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
ud._completed_chunks = mock.MagicMock()
ud._src_ase = ase
assert ud._resume() == 4
# check no encryption
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
opts.rsa_public_key = 'abc'
nc = mock.MagicMock()
nc.length = 16
nc.completed = False
nc.total_chunks = 2
nc.chunk_size = 1
nc.completed_chunks = 1
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() is None
# check rr path exists
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
nc.length = 2
nc.local_path = pathlib.Path('yyy')
opts.rsa_public_key = None
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() is None
# check resume no md5
opts.store_file_properties.md5 = False
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
nc = mock.MagicMock()
nc.length = 2
nc.completed = False
nc.total_chunks = 2
nc.chunk_size = 1
cc = bitstring.BitArray(length=nc.total_chunks)
cc.set(True, 0)
nc.completed_chunks = cc.int
nc.local_path = lp.absolute_path
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() == 1
# check resume with md5 mismatch
opts.store_file_properties.md5 = True
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
nc = mock.MagicMock()
nc.length = 2
nc.completed = False
nc.total_chunks = 2
nc.chunk_size = 1
cc = bitstring.BitArray(length=nc.total_chunks)
cc.set(True, 0)
nc.completed_chunks = cc.int
nc.local_path = lp.absolute_path
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() is None
# check resume with md5 match
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
nc = mock.MagicMock()
nc.length = 2
nc.completed = False
nc.total_chunks = 2
nc.chunk_size = 1
cc = bitstring.BitArray(length=nc.total_chunks)
cc.set(True, 0)
nc.completed_chunks = cc.int
nc.local_path = lp.absolute_path
md5 = hashlib.md5()
md5.update(b'z')
nc.md5hexdigest = md5.hexdigest()
resume.get_record.return_value = nc
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(), resume)
assert ud._resume() == 1
import pathlib2
HERE = pathlib2.Path(__file__).resolve().parent
FROZEN = HERE.joinpath('frozen.txt')
FROZEN_DOC = HERE.joinpath('frozen_docs.txt')
def test_match_files():
with FROZEN.open() as fp:
frozen = dict(x.strip().split('==') for x in fp)
with FROZEN_DOC.open() as fp:
frozen_doc = dict(x.strip().split('==') for x in fp)
expected_frozen_doc = {k: v for k, v in frozen.items() if k in frozen_doc}
assert expected_frozen_doc == frozen_doc
def test_localsourcepaths_files(tmpdir):
tmpdir.mkdir('abc')
tmpdir.join('moo.cow').write('z')
abcpath = tmpdir.join('abc')
abcpath.join('hello.txt').write('hello')
abcpath.join('blah.x').write('x')
abcpath.join('blah.y').write('x')
abcpath.join('blah.z').write('x')
abcpath.mkdir('def')
defpath = abcpath.join('def')
defpath.join('world.txt').write('world')
defpath.join('moo.cow').write('y')
a = upload.LocalSourcePath()
a.add_includes('**')
a.add_includes('*.txt')
a.add_includes(('moo.cow', '*blah*'))
with pytest.raises(ValueError):
a.add_includes('**/**/*')
a.add_excludes('**')
a.add_excludes('**/blah.x')
with pytest.raises(ValueError):
a.add_excludes('**/**/blah.x')
a.add_excludes(['world.txt'])
a.add_path(str(tmpdir))
a_set = set()
for file in a.files():
sfile = str(file.parent_path / file.relative_path)
a_set.add(sfile)
assert len(a._include) == 3
assert len(a._exclude) == 2
assert not a.can_rename()
assert len(a.paths) == 1
assert str(abcpath.join('blah.x')) in a_set
assert str(defpath.join('world.txt')) in a_set
assert str(defpath.join('moo.cow')) not in a_set
b = upload.LocalSourcePath()
b.add_includes(['moo.cow', '*blah*'])
b.add_includes('*.txt')
b.add_excludes(('world.txt', ))
b.add_excludes('**/blah.x')
b.add_paths([pathlib.Path(str(tmpdir))])
for file in a.files():
sfile = str(file.parent_path / file.relative_path)
assert sfile in a_set
assert upload.LocalSourcePath.is_stdin('-')
assert upload.LocalSourcePath.is_stdin('/dev/stdin')
assert not upload.LocalSourcePath.is_stdin('/')
c = upload.LocalSourcePath()
c.add_path('-')
for file in c.files():
assert file.use_stdin
d = upload.LocalSourcePath()
d.add_path(str(tmpdir.join('moo.cow')))
i = 0
for file in d.files():
assert str(file.parent_path.absolute()) == str(tmpdir)
assert str(file.relative_path) == 'moo.cow'
assert not file.use_stdin
i += 1
assert i == 1
tmpdir.join('moo.cow2').ensure(file=True)
d.add_path(str(tmpdir.join('moo.cow2')))
i = 0
for file in d.files():
i += 1
assert i == 2
napoleon_google_docstring = False
napoleon_use_param = True
napoleon_use_ivar = True
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
source_suffix = ['.rst', '.md']
master_doc = 'index'
# General information about the project.
project = 'toppra'
copyright = '2020, Hung Pham'
author = 'Hung Pham'
version = pathlib.Path('./../../VERSION').read_text()
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', '**.ipynb_checkpoints']
# pygments_style = 'friendly'
def test_descriptor_next_offsets(tmpdir):
tmpdir.join('a').write('ab')
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts = mock.MagicMock()
opts.chunk_size_bytes = 1
opts.one_shot_bytes = 0
opts.store_file_properties.md5 = True
opts.rsa_public_key = None
ase = azmodels.StorageEntity('cont')
ase._mode = azmodels.StorageModes.Block
ase._name = 'name'
ase._encryption = None
# test normal
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._resume = mock.MagicMock()
ud._resume.return_value = None
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 0
assert offsets.num_bytes == 1
assert offsets.range_start == 0
assert offsets.range_end == 0
assert not offsets.pad
assert ud._offset == 1
assert ud._chunk_num == 1
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 1
assert offsets.num_bytes == 1
assert offsets.range_start == 1
assert offsets.range_end == 1
assert not offsets.pad
assert ud._offset == 2
assert ud._chunk_num == 2
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets is None
# test chunk size exceeds size
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts.chunk_size_bytes = 3
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._chunk_size = 3
ud._resume = mock.MagicMock()
ud._resume.return_value = None
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 0
assert offsets.num_bytes == 2
assert offsets.range_start == 0
assert offsets.range_end == 1
assert not offsets.pad
assert ud._offset == 2
assert ud._chunk_num == 1
# test encrypted
tmpdir.join('a').write('z' * 16)
lp = upload.LocalPath(pathlib.Path(str(tmpdir)), pathlib.Path('a'))
opts.chunk_size_bytes = 16
opts.rsa_public_key = 'abc'
ud = upload.Descriptor(lp, ase, 'uid', opts, mock.MagicMock(),
mock.MagicMock())
ud._resume = mock.MagicMock()
ud._resume.return_value = None
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 0
assert offsets.num_bytes == 16
assert offsets.range_start == 0
assert offsets.range_end == 15
assert not offsets.pad
assert ud._offset == 16
assert ud._chunk_num == 1
offsets, rb = ud.next_offsets()
assert rb is None
assert offsets.chunk_num == 1
assert offsets.num_bytes == 16
assert offsets.range_start == 16
assert offsets.range_end == 31
assert offsets.pad
assert ud._offset == 32
assert ud._chunk_num == 2
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--device-model-id',
'--device_model_id',
type=str,
metavar='DEVICE_MODEL_ID',
required=False,
help='the device model ID registered with Google')
parser.add_argument('--project-id',
'--project_id',
type=str,
metavar='PROJECT_ID',
required=False,
help='the project ID used to register this device')
parser.add_argument('--device-config',
type=str,
metavar='DEVICE_CONFIG_FILE',
default=os.path.join(os.path.expanduser('~/.config'),
'googlesamples-assistant',
'device_config_library.json'),
help='path to store and read device configuration')
parser.add_argument('--credentials',
type=existing_file,
metavar='OAUTH2_CREDENTIALS_FILE',
default=os.path.join(os.path.expanduser('~/.config'),
'google-oauthlib-tool',
'credentials.json'),
help='path to store and read OAuth2 credentials')
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s ' + Assistant.__version_str__())
args = parser.parse_args()
with open(args.credentials, 'r') as f:
credentials = google.oauth2.credentials.Credentials(token=None,
**json.load(f))
device_model_id = None
last_device_id = None
try:
with open(args.device_config) as f:
device_config = json.load(f)
device_model_id = device_config['model_id']
last_device_id = device_config.get('last_device_id', None)
except FileNotFoundError:
pass
if not args.device_model_id and not device_model_id:
raise Exception('Missing --device-model-id option')
# Re-register if "device_model_id" is given by the user and it differs
# from what we previously registered with.
should_register = (args.device_model_id
and args.device_model_id != device_model_id)
device_model_id = args.device_model_id or device_model_id
with Assistant(credentials, device_model_id) as assistant:
events = assistant.start()
device_id = assistant.device_id
print('device_model_id:', device_model_id)
print('device_id:', device_id + '\n')
# Re-register if "device_id" is different from the last "device_id":
if should_register or (device_id != last_device_id):
if args.project_id:
register_device(args.project_id, credentials, device_model_id,
device_id)
pathlib.Path(os.path.dirname(
args.device_config)).mkdir(exist_ok=True)
with open(args.device_config, 'w') as f:
json.dump(
{
'last_device_id': device_id,
'model_id': device_model_id,
}, f)
else:
print(WARNING_NOT_REGISTERED)
for event in events:
process_event(event, assistant)
def _get_components_to_download(self,
component_path=None,
download_to_path=None):
"""
Identify components to be downloaded, along with their local destination paths.
Parameters
----------
component_path : str, optional
Path to directory or file within blob.
download_to_path : str, optional
Local path to download to.
Returns
-------
components_to_download : dict
Map of component paths to local destination paths.
downloaded_to_path : str
Absolute path where file(s) were downloaded to. Matches `download_to_path` if it was
provided as an argument.
"""
implicit_download_to_path = download_to_path is None
if component_path is not None:
# look for an exact match with `component_path` as a file
for path in self.list_paths():
if path == component_path:
if implicit_download_to_path:
# default to filename from `component_path`, in cwd
local_path = os.path.basename(component_path)
# avoid collision with existing file
local_path = _file_utils.without_collision(local_path)
else:
# exactly where the user requests
local_path = download_to_path
return ({path: local_path}, os.path.abspath(local_path))
# no exact match, so it's a folder download (or nonexistent path)
# figure out where files are going to be downloaded to
if implicit_download_to_path:
if component_path is None:
downloaded_to_path = DEFAULT_DOWNLOAD_DIR
# avoid collision with existing directory
downloaded_to_path = _file_utils.without_collision(
downloaded_to_path)
else: # need to automatically determine directory
# NOTE: if `component_path` == "s3://" with any trailing slashes, it becomes "s3:"
downloaded_to_path = pathlib2.Path(
component_path).name # final path component
if downloaded_to_path in {".", "..", "/", "s3:"}:
# rather than dump everything into cwd, use new child dir
downloaded_to_path = DEFAULT_DOWNLOAD_DIR
# avoid collision with existing directory
downloaded_to_path = _file_utils.without_collision(
downloaded_to_path)
else:
# exactly where the user requests
downloaded_to_path = download_to_path
# collect paths in blob and map them to download locations
components_to_download = dict()
if component_path is None:
# download all
for path in self.list_paths():
local_path = os.path.join(
downloaded_to_path,
_file_utils.remove_prefix_dir(path, "s3:"),
)
components_to_download[path] = local_path
else:
# look for files contained in `component_path` as a directory
component_path_as_dir = component_path if component_path.endswith(
'/') else component_path + '/'
for path in self.list_paths():
if path.startswith(component_path_as_dir):
# rebase from `component_path` onto `downloaded_to_path`
# Implicit `download_to_path` example:
# component_blob.path = "coworker/downloads/data/info.csv"
# component_path = "coworker/downloads"
# downloaded_to_path = "downloads" or "downloads 1", etc.
# local_path = "downloads/data/info.csv"
# Explicit `download_to_path` example:
# component_blob.path = "coworker/downloads/data/info.csv"
# component_path = "coworker/downloads"
# downloaded_to_path = "my-data"
# local_path = "my-data/data/info.csv"
local_path = os.path.join(
downloaded_to_path,
_file_utils.remove_prefix_dir(
path, prefix_dir=component_path),
)
components_to_download[path] = local_path
if not components_to_download:
raise KeyError(
"no components found for path {}".format(component_path))
return (components_to_download, os.path.abspath(downloaded_to_path))
