def get(name=None):
"""
API endpoint for returning a given device by its name, all if name is not specified.
:return: All devices.
"""
try:
for attempt in range(1, RETRIEVE_RETRY + 1):
log.info(f'Attempt [{attempt}]...')
chromecast_list = pychromecast.get_chromecasts()
chromecast_list = [
Device(chromecast_item) for chromecast_item in chromecast_list
]
if name is not None:
log.info(f'Applying filter name by name: [{name}]')
chromecast_list = [
device for device in chromecast_list
if device.has_device_name(name)
]
chromecast_list = [
device.serialize() for device in chromecast_list
]
if chromecast_list:
return response.make(
error=False,
response=dict(chromecast_list=chromecast_list))
else:
log.warn(f'No devices found at attempt {attempt}.')
time.sleep(RETRIEVE_RTD)
log.warn(f'No devices found.')
return response.make(error=False, response=dict(chromecast_list=[]))
except Exception as e:
log.error(f'Unexpected error: [{e}]')
log.exception(e)
return response.make(error=True, message=MESSAGE_UNEXPECTED_ERROR)
def run():
"""
Main function in order to process a new iteration of the Twitter bot.
:return: Iteration done.
"""
try:
for i, item in enumerate(RESOURCES):
item_name = item['name']
log.info(f'{i + 1}/{len(RESOURCES)} Processing {item_name}...')
results = retriever.get_last_update(item['data_url'])
if results is not None:
item_data_path = item['data_path']
item_data_path_exists = os.path.isfile(item_data_path)
# Get old results if they exist
old_results = dict()
if item_data_path_exists:
with open(item_data_path, 'r') as item_data_file:
old_results = json.load(item_data_file)
# Save latest results
with open(item_data_path, 'w') as item_data_file:
json.dump(results, item_data_file)
# Check for differences if it did not exist before
if item_data_path_exists:
total_worldwide = sum(old_results.values())
diff_results = list(dictdiffer.diff(results, old_results))
for j, diff_tuple in enumerate(diff_results):
log.info(f'{j + 1}/{len(diff_results)} New changes found: [{diff_tuple}]')
total_worldwide = _notify_changes(diff_tuple, item_name, item['icon'], results, total_worldwide)
time.sleep(TIME_BETWEEN_RESOURCES)
except Exception as e:
log.error(f'Unexpected error: [{e}]')
log.exception(e)
def run():
"""
Main function for running the bot.
:return: Iteration done.
"""
log.info('Job started!')
runner.run()
log.info('Job ended!')
def load(h5_file_path, h5_type, cut=0):
log.info(f'Loading {h5_type}...')
with h5py.File(h5_file_path, 'r') as f:
x_train = _cut(f['x_train'], cut)
if TRAIN_STRATEGY == 'relu':
x_train = x_train.reshape(x_train.shape[0], TRAIN_INPUT_SIZE, TRAIN_INPUT_SIZE, 3)
x_train = x_train.astype('float32') / 255
y_train = to_categorical(_cut(f['y_train'], cut), num_classes=len(DATASET_CATEGORIES))
log.info(f'{h5_type.capitalize()} items: {x_train.shape[0]}')
return x_train, y_train
def tweet(message_list):
"""
Tweet a bunch of messages to Twitter with some time between them.
:param message_list: Message list.
:return: All messages tweeted.
"""
for message in message_list:
log.info(message)
if not DEBUG:
api.PostUpdate(message)
time.sleep(TIME_BETWEEN_TWEETS)
def run():
"""
Main function in order to process a new iteration of the Twitter bot.
:return: Iteration done.
"""
try:
resources_list = [DATA_CONFIRMED, DATA_DEATHS, DATA_RECOVERED]
confirmed_results, deaths_results, recovered_results = retriever.get_last_update(
)
if all([confirmed_results, deaths_results, recovered_results]):
for i, item_name in enumerate(resources_list):
log.info(
f'{i + 1}/{len(resources_list)} Processing {item_name}...')
# Get results
results = None
if item_name == DATA_CONFIRMED:
results = confirmed_results
elif item_name == DATA_DEATHS:
results = deaths_results
elif item_name == DATA_RECOVERED:
results = recovered_results
# Process
if results is not None:
item_data_path = DATA_PATH_DICT[item_name]
item_data_path_exists = os.path.isfile(item_data_path)
# Get old results if they exist
old_results = dict()
if item_data_path_exists:
with open(item_data_path, 'r') as item_data_file:
old_results = json.load(item_data_file)
# Save latest results
with open(item_data_path, 'w') as item_data_file:
json.dump(results, item_data_file)
# Check for differences if it did not exist before
if item_data_path_exists:
total_worldwide = sum(old_results.values())
diff_results = list(
dictdiffer.diff(results, old_results))
for j, diff_tuple in enumerate(diff_results):
log.info(
f'{j + 1}/{len(diff_results)} New changes found: [{diff_tuple}]'
)
total_worldwide = _notify_changes(
diff_tuple, item_name, ICON_DICT[item_name],
results, total_worldwide)
time.sleep(TIME_BETWEEN_RESOURCES)
except Exception as e:
log.error(f'Unexpected error: [{e}]')
log.exception(e)
def main(preprocessed_folder, training_data, validation_data):
os.makedirs(preprocessed_folder, exist_ok=True)
t_folder_list = [
d for d in os.listdir(training_data)
if os.path.isdir(os.path.join(training_data, d))
]
v_folder_list = [
d for d in os.listdir(validation_data)
if os.path.isdir(os.path.join(validation_data, d))
]
for name, path in [(DATASET_SUB_FOLDER_TRAINING, t_folder_list),
(DATASET_SUB_FOLDER_VALIDATION, v_folder_list)]:
log.info(name)
x_data = list()
y_data = list()
for category_name in tqdm(path, total=len(path)):
stl_array = list()
for thing_folder in os.listdir(
os.path.join(training_data, category_name)):
thing_folder = os.path.join(training_data, category_name,
thing_folder)
stl_array.append([
os.path.join(thing_folder, s)
for s in os.listdir(thing_folder)
])
with ThreadPool(STANDARDIZE_CONCURRENCY) as pool:
category_things = list(
tqdm(pool.imap(extractor.extract_features,
stl_array,
chunksize=1),
total=len(stl_array)))
category_things = [c for c in category_things if c is not None]
x_data.extend(category_things)
y_data.extend(
[list(DATASET_CATEGORIES.keys()).index(category_name)] *
len(category_things))
x_data = np.array(x_data)
y_data = np.array(y_data)
with h5py.File(os.path.join(preprocessed_folder, f'{name}.h5'),
'w') as reduced_features_file:
reduced_features_file.create_dataset('x_train', data=x_data)
reduced_features_file.create_dataset('y_train', data=y_data)
def main():
if not os.path.isfile('.env'):
log.error('Environment file (.env) could not be found.')
return
log.info(f'Spinning up local API for extracting Thingiverse Thing IDs at port {THINGIVERSE_FLASK_PORT}.')
process = multiprocessing.Process(target=launcher.run_app, args=())
process.start()
log.info(f'Sleeping {THINGIVERSE_FLASK_WAIT_PRE} seconds for letting the API to initialize and open a browser.')
time.sleep(THINGIVERSE_FLASK_WAIT_PRE)
webbrowser.open(f'http://localhost:{THINGIVERSE_FLASK_PORT}{THINGIVERSE_FLASK_ENDPOINT}', new=2)
if THINGIVERSE_FLASK_WAIT_ENABLE:
log.info(f'Sleeping {THINGIVERSE_FLASK_WAIT_POST} seconds for letting the process to finish.')
log.info('You can stop the script if the process is done.')
time.sleep(THINGIVERSE_FLASK_WAIT_POST)
log.info('Killing processes.')
process.terminate()
else:
log.debug('Processing forever.')
def main(weights_name, weights_folder, validation_h5, cut, verbose):
model_path = os.path.join(weights_folder, weights_name, TRAIN_MODEL_FILE)
assert os.path.isfile(model_path)
x_train, y_train = loader.load(validation_h5, DATASET_SUB_FOLDER_VALIDATION, cut=cut)
model = tf.keras.models.load_model(model_path)
model.summary()
loss, acc = model.evaluate(x_train, y_train, verbose=2)
log.info('Restored model, accuracy: {:5.2f}%'.format(100 * acc))
main_category_achieved = list()
for pred, gt in zip(model.predict(x_train), y_train):
pred_idx = int(np.argmax(pred))
label = list(DATASET_CATEGORIES.keys())[pred_idx]
confidence = float(pred[pred_idx])
gt_label = list(DATASET_CATEGORIES.keys())[int(np.argmax(gt))]
achieved = label == gt_label
if verbose:
log.info('{}: {:5.2f}% -> {}{}'.format(label, 100 * confidence, gt_label, ' | YAY' if achieved else ''))
main_category_achieved.append(
DATASET_CATEGORIES[label]['main_category'] == DATASET_CATEGORIES[gt_label]['main_category']
)
log.info('Main category accuracy: {:5.2f}%'.format(100 * sum(main_category_achieved) / len(main_category_achieved)))
def _count(training_folder, validation_folder):
it = [(DATASET_SUB_FOLDER_TRAINING, training_folder),
(DATASET_SUB_FOLDER_VALIDATION, validation_folder)]
for name, path in it:
total = 0
log.info(name.upper())
for category_name in os.listdir(path):
category_folder = os.path.join(path, category_name)
if os.path.isdir(category_folder):
category_count = len(glob.glob(f'{category_folder}/*/*'))
total += category_count
log.info('{}: {} examples'.format(category_name,
category_count))
log.info('total: {} examples'.format(total))
def __exit__(self, *args):
self.end = time.time()
self.time = self.end - self.start
log.info(f'{self.name}: {self.time:.6f}s')
import time
from src import scheduler
from src.helper import log, env
if __name__ == '__main__':
try:
log.info('auto-factorial was born!')
if not env.is_development():
scheduler.scheduler.start()
else:
log.info(
'Running jobs manually since DEVELOPMENT MODE is enabled, and then sleep eternally.'
)
scheduler.auto_factorial_cron_job()
while True:
time.sleep(1000)
except Exception as e:
log.error(f'Unexpected error {e} in auto-factorial')
log.exception(e)
finally:
log.info('Rest in peace, auto-factorial.')
def get(name, source_url):
"""
API endpoint for playing a source to a device given its name.
:param name: Device name.
:param source_url: Source URL.
:return: Source playing at given device.
"""
try:
# Random generation, if needed
random_video = None
if source_url == PLAY_RANDOM:
log.info('Getting random video...')
random_video = random_extractor.get()
source_url = random_video.get('sources')[0]
# Extracting information
log.info('Extracting source names...')
source_ending = '.{}'.format(source_url.split('.')[-1])
source_name = source_url.split(
'/')[-1] if not random_video else random_video.get('title')
log.info(f'Source ending: [{source_ending}]')
log.info(f'Source name: [{source_name}]')
if source_ending not in MIME_TYPES:
log.warn(MESSAGE_PLAY_DEVICE_NOT_FOUND)
return response.make(error=True,
message=MESSAGE_PLAY_DEVICE_NOT_FOUND)
# Getting devices
for attempt in range(1, RETRIEVE_RETRY + 1):
log.info(f'Attempt [{attempt}]...')
chromecast_list = pychromecast.get_chromecasts()
chromecast_list = [
Device(chromecast_item) for chromecast_item in chromecast_list
]
chromecast_list = [
device for device in chromecast_list
if device.has_device_name(name)
]
if chromecast_list:
# Process media
chromecast_device = chromecast_list[0]
log.info(
f'Chromecast device: [{chromecast_device.uuid} - {chromecast_device.name}]'
)
log.info('Waiting for being loaded...')
chromecast_device.chromecast_object.wait()
log.info('Loaded.')
mime_type = MIME_TYPES[source_ending]
log.info(f'MIME type: [{mime_type}]')
log.info(f'Cast type: [{chromecast_device.cast_type}]')
if (chromecast_device.cast_type == 'audio' and 'video' in mime_type) or \
(chromecast_device.cast_type == 'group' and 'video' in mime_type):
log.warn(MESSAGE_PLAY_MIME_DEVICE_NOT_SUPPORTED)
return response.make(
error=True,
message=MESSAGE_PLAY_MIME_DEVICE_NOT_SUPPORTED)
# Play media
log.info('Playing media...')
chromecast_device.chromecast_object.media_controller.play_media(
url=source_url, content_type=mime_type, title=source_name)
chromecast_device.chromecast_object.media_controller.block_until_active(
)
log.info('Done!')
return response.make(
error=False, response=dict(message=MESSAGE_PLAY_SUCCESS))
else:
log.warn(f'No devices found at attempt {attempt}.')
time.sleep(RETRIEVE_RTD)
log.warn(f'No devices found.')
return response.make(error=True, message=MESSAGE_PLAY_DEVICE_NOT_FOUND)
except Exception as e:
log.error(f'Unexpected error: [{e}]')
log.exception(e)
return response.make(error=True, message=MESSAGE_UNEXPECTED_ERROR)
import time
from src import scheduler
from src.config import DEBUG
from src.helper import log
if __name__ == '__main__':
try:
log.info('Covid19-bot was born!')
if not DEBUG:
scheduler.scheduler.start()
else:
log.info('Running jobs manually since DEBUG is enabled.')
time.sleep(5)
scheduler.run()
while True:
time.sleep(10000)
except Exception as e:
log.error(f'Unexpected error {e} in Covid-19')
log.exception(e)
finally:
log.info('Rest in peace, Covid-19.')
def __exit__(self, *args):
self.end = time.time()
self.time = self.end - self.start
log.info('{}: {:.6f}s'.format(self.name, self.time))
