def process(config, delete_after, startswith): yt_config = config['youtube'] provider = get_provider_obj(config) project_dir = os.path.join(DOWNLOADS_DIR, config['id']) for download_dir in filter(lambda d: not d.startswith('_'), sorted(os.listdir(project_dir))): if startswith and not download_dir.startswith(startswith): continue download_dir = os.path.join(project_dir, download_dir) if not os.path.isdir(download_dir): continue metadatas = yaml_load(os.path.join(download_dir, '_metadata.yaml')) mono = config.get('audio_mono', False) for video_metadata in metadatas: prepped_video_info = provider.postprocess(video_metadata, download_dir, VIDEOS_DIR, mono=mono) prepped_video_info.config_id = config['id'] overrides = tweak_metadata(config['id'], prepped_video_info.video_metadata) subs = build_substitutions_dict(prepped_video_info.video_metadata) subs.update(overrides) prepped_video_info.title = yt_config['title'].format(**subs) prepped_video_info.description = yt_config['desc'].format(**subs).strip() prepped_video_info.playlists = [pl.format(**subs) for pl in yt_config['playlists']] prepped_video_info_path = os.path.join(VIDEOS_DIR, prepped_video_info.video_filename + '.yaml') yaml_dump(prepped_video_info, prepped_video_info_path) print("Updated " + prepped_video_info_path) if delete_after: print("Deleting " + project_dir) shutil.rmtree(project_dir)
def s3(config, delete_after): FileUpload = namedtuple('FileUpload', ['src_path', 's3_key']) files_to_upload = [] if config['id'] == 'coquitlam': # Upload unprocessed videos for Coquitlam. download_dir = os.path.join(DOWNLOADS_DIR, config['id']) for subdir in os.listdir(download_dir): subdir = os.path.join(download_dir, subdir) if not os.path.isdir(subdir): continue metadata_path = os.path.join(subdir, '_metadata.yaml') meta = yaml_load(metadata_path)[0] video_filename = os.path.basename(meta.url) video_path = os.path.join(subdir, video_filename) s3_base_path = '{}/{}/'.format(config['id'], meta.start_ts.split('-')[0]) if os.path.exists(video_path): files_to_upload.extend([ FileUpload(metadata_path, s3_base_path + video_filename + '.metadata.yaml'), FileUpload(video_path, s3_base_path + video_filename), ]) else: for metadata_filename in sorted(filter(lambda f: f.endswith('.yaml'), os.listdir(VIDEOS_DIR))): metadata_path = os.path.join(VIDEOS_DIR, metadata_filename) prepped_video_info = yaml_load(metadata_path) if prepped_video_info.config_id != config['id']: continue video_path = os.path.join(VIDEOS_DIR, prepped_video_info.video_filename) if not os.path.exists(video_path): print(video_path + " doesn't exist") continue s3_base_path = '{}/{}/'.format(config['id'], prepped_video_info.video_metadata.clip_start_utc.split('-')[0]) files_to_upload.extend([ FileUpload(metadata_path, s3_base_path + metadata_filename), FileUpload(video_path, s3_base_path + prepped_video_info.video_filename), ]) client = S3Transfer(boto3.client('s3')) bucket = config['s3_bucket'] for entry in files_to_upload: print("Uploading {} to bucket {} at key {}".format(entry.src_path, bucket, entry.s3_key)) client.upload_file(entry.src_path, bucket, entry.s3_key) if delete_after: print("Deleting " + entry.src_path) os.remove(entry.src_path)
def classinfo_from_modelinfo(mongodb, dbname, filepath): modelinfo = common.yaml_load(filepath) log.info("modelinfo: {}".format(modelinfo)) lbl_ids = modelinfo['classes'] classinfo = [{ 'lbl_id': lbl_id, 'source': 'hmd', 'name': lbl_id } for lbl_id in lbl_ids[1:]] tblname = annonutils.get_tblname('CLASSINFO') mongodb.connect(dbname) mongodb.write(tblname, classinfo, idx_col='lbl_id')
def filter_data(df): """ Filtering data from errors and outliers """ df['tpep_pickup_datetime'] = df['tpep_pickup_datetime'].astype( 'datetime64') df['tpep_dropoff_datetime'] = df['tpep_dropoff_datetime'].astype( 'datetime64') row_len = len(df) # Create duration column df['trip_duration_min'] = (df['tpep_dropoff_datetime'] - df['tpep_pickup_datetime']) / \ np.timedelta64(1, 'm') # Arround minute df['trip_duration_min'] = df['trip_duration_min'].apply(round_min) # Zero duration cleaning. Threshold < 1 minute df.drop(df[df['trip_duration_min'] < 1].index, inplace=True, axis=0) # Zero passenger cleaning. Threshold = 0 passenger df.drop(df[df['passenger_count'] == 0].index, inplace=True) # Zero distance cleaning. Threshold < 1 mile distance df.drop(df[df['trip_distance'] < 1].index, inplace=True) # only town region town_square = common.yaml_load()['ny_square'] d_latitude, d_longitude = get_grid_params(grid_input=town_square, grid_cells=50) df['region'] = df.apply(get_region_5, d_latitude=d_latitude, d_longitude=d_longitude, ny_grid=town_square, grid_cells=50, axis=1) df.drop(df[df['region'] > 2500].index, inplace=True) logging.debug(f"Has been filtered {row_len - len(df)} rows") return df
def upload(config, delete_after): yt_config = config['youtube'] client_creds = load_client_credentials() tokens_file = tokens_file_for_id(config['id']) if not os.path.isfile(tokens_file): print("Need to perform 'youtube authorize' first.") with open(tokens_file) as inf: tokens = json.load(inf) session = YouTubeSession(client_creds, tokens) for metadata_filename in sorted(filter(lambda f: f.endswith('.yaml'), os.listdir(VIDEOS_DIR))): metadata_path = os.path.join(VIDEOS_DIR, metadata_filename) prepped_video_info = yaml_load(metadata_path) if prepped_video_info.config_id != config['id']: continue video_path = os.path.join(VIDEOS_DIR, prepped_video_info.video_filename) if not os.path.exists(video_path): print(video_path + " doesn't exist") continue yt_video_res = build_youtube_resource( prepped_video_info.title, prepped_video_info.description, pendulum.parse(prepped_video_info.video_metadata.start_ts), coords=yt_config['location'], location_desc=yt_config['location_desc'], tags=yt_config['tags'], privacy=yt_config['privacy'], ) print(yt_video_res) yt_video_id = session.upload(video_path, yt_video_res, yt_config['notify_subscribers']) for playlist in prepped_video_info.playlists: session.add_video_to_playlist(playlist, yt_video_id, yt_config['privacy']) if delete_after: for f in (metadata_path, video_path): print("Deleting " + f) os.remove(f)
import pandas as pd import numpy as np import seaborn as sns import matplotlib.pyplot as plt # original lib import common ######################################################################## ######################################################################## # configuration and logs ######################################################################## logging.basicConfig(level=logging.DEBUG, filename='logs.log', format='%(asctime)s %(levelname)s:%(message)s') path = common.yaml_load()['taxi_data'] ######################################################################## def round_min(minute): return round(minute, 2) def get_grid_params(grid_input: dict, grid_cells: int): d_latitude = (grid_input['latitude'][1] - grid_input['latitude'][0]) / grid_cells d_longitude = (grid_input['longitude'][1] - grid_input['longitude'][0]) / grid_cells return d_latitude, d_longitude
import librosa import pandas as pd from tqdm import tqdm from scipy.io.wavfile import write # original lib import common ######################################################################## ######################################################################## # configuration and logs ######################################################################## logging.basicConfig(level=logging.DEBUG, filename='logs.log', format='%(asctime)s %(levelname)s:%(message)s') param = common.yaml_load() iter_normal = ['normal', 'anomaly'] iter_type = {'audio': 'wav', 'vibro': 'txt', 'modbus': 'csv'} ######################################################################## def file_name_list(dir, dir_name, prefix_normal, prefix_type, ext): """ Return file name with extention in path """ file_name_list = sorted( glob.glob( "{dir}/{dir_name}/{prefix_normal}/{prefix_type}/*.{ext}".format( dir=dir,
def load_date_metadata(): for dt in for_dates: date_metadata_path = os.path.join(metadata_dir, dt.to_date_string() + '.yaml') if not os.path.exists(date_metadata_path): raise ValueError("No metadata downloaded for " + dt.to_date_string()) yield yaml_load(date_metadata_path)
######################################################################## ######################################################################## # import additional python-library ######################################################################## import numpy from sklearn import metrics import common as com from pytorch_model import VariationalAutoEncoder as VAE import torch.utils.data ######################################################################## ######################################################################## # load parameter.yaml ######################################################################## PARAM = com.yaml_load('./gan.yaml') ####################################################################### ######################################################################## # def ######################################################################## def save_csv(save_file_path, save_data): with open(save_file_path, "w", newline="") as f: writer = csv.writer(f, lineterminator='\n') writer.writerows(save_data) def get_machine_id_list_for_test(target_dir, dir_name="test", ext="wav"): """
# from import from tqdm import tqdm from sklearn import metrics try: from sklearn.externals import joblib except: import joblib # original lib import common as com import keras_model ######################################################################## ######################################################################## # load parameter.yaml ######################################################################## param = com.yaml_load() ####################################################################### ######################################################################## # output csv file ######################################################################## def save_csv(save_file_path, save_data): with open(save_file_path, "w", newline="") as f: writer = csv.writer(f, lineterminator='\n') writer.writerows(save_data) ########################################################################
from vibro.vibro import vibro_write from modbus.adapter import modbus_write ######################################################################## ######################################################################## # import interface library ######################################################################## from tkinter import Tk, Button from threading import Thread from tkinter import messagebox as mb from tkinter import filedialog as fd from tkinter.filedialog import askopenfilename ######################################################################## #param loading param = yaml_load("config.yaml") audio_param = param['audio'] vibro_param = param['vibro'] modbus_param = param['modbus'] flagCOMPortRead = False serialCOMPort = serial.Serial() #main funcrions def stop(): global flagCOMPortRead flagCOMPortRead = False print('Stop') serialCOMPort.close() writeThread.join()
######################################################################## # import additional python-library ######################################################################## import numpy import common as com import torch import torch.utils.data from torch import optim, nn from torch.utils.data.dataset import Subset from pytorch_model import AutoEncoder ######################################################################## # load parameter.yaml ######################################################################## param = com.yaml_load('./baseline.yaml') ######################################################################## ######################################################################## # Dataset ######################################################################## class dcaseDataset(torch.utils.data.Dataset): def __init__(self, target_dir, dir_name="train", ext="wav", n_mels=64, frames=5, n_fft=1024,
######################################################################## # import additional python-library ######################################################################## import numpy from sklearn import metrics import common as com from pytorch_model import VariationalAutoEncoder as VAE import torch.utils.data ######################################################################## ######################################################################## # load parameter.yaml ######################################################################## PARAM = com.yaml_load('./vae.yaml') ####################################################################### ######################################################################## # def ######################################################################## def save_csv(save_file_path, save_data): with open(save_file_path, "w", newline="") as f: writer = csv.writer(f, lineterminator='\n') writer.writerows(save_data) def get_machine_id_list_for_test(target_dir, dir_name="test",
sys.path.append('..') import common as com import pytorch_common from model import * ######################################################################## ######################################################################## # control stochastic behavior ######################################################################## deterministic_everything(2022, pytorch=True) ####################################################################### ######################################################################## # load parameter.yaml ######################################################################## params = EasyDict(com.yaml_load('config.yaml')) ####################################################################### ######################################################################## # def ######################################################################## def save_csv(save_file_path, save_data): with open(save_file_path, "w", newline="") as f: writer = csv.writer(f, lineterminator='\n') writer.writerows(save_data) def get_machine_id_list_for_test(target_dir, dir_name="test", ext="wav"): """
sys.path.append('..') import common as com import pytorch_common import model ######################################################################## ######################################################################## # control stochastic behavior ######################################################################## deterministic_everything(2022, pytorch=True) ####################################################################### ######################################################################## # load parameter.yaml ######################################################################## param = com.yaml_load('config.yaml') ####################################################################### ######################################################################## # def ######################################################################## def save_csv(save_file_path, save_data): with open(save_file_path, "w", newline="") as f: writer = csv.writer(f, lineterminator='\n') writer.writerows(save_data) def get_machine_id_list_for_test(target_dir, dir_name="test", ext="wav"): """