def process(config, delete_after, startswith):
yt_config = config['youtube']
provider = get_provider_obj(config)
project_dir = os.path.join(DOWNLOADS_DIR, config['id'])
for download_dir in filter(lambda d: not d.startswith('_'), sorted(os.listdir(project_dir))):
if startswith and not download_dir.startswith(startswith):
continue
download_dir = os.path.join(project_dir, download_dir)
if not os.path.isdir(download_dir):
continue
metadatas = yaml_load(os.path.join(download_dir, '_metadata.yaml'))
mono = config.get('audio_mono', False)
for video_metadata in metadatas:
prepped_video_info = provider.postprocess(video_metadata, download_dir, VIDEOS_DIR, mono=mono)
prepped_video_info.config_id = config['id']
overrides = tweak_metadata(config['id'], prepped_video_info.video_metadata)
subs = build_substitutions_dict(prepped_video_info.video_metadata)
subs.update(overrides)
prepped_video_info.title = yt_config['title'].format(**subs)
prepped_video_info.description = yt_config['desc'].format(**subs).strip()
prepped_video_info.playlists = [pl.format(**subs) for pl in yt_config['playlists']]
prepped_video_info_path = os.path.join(VIDEOS_DIR, prepped_video_info.video_filename + '.yaml')
yaml_dump(prepped_video_info, prepped_video_info_path)
print("Updated " + prepped_video_info_path)
if delete_after:
print("Deleting " + project_dir)
shutil.rmtree(project_dir)
def s3(config, delete_after):
FileUpload = namedtuple('FileUpload', ['src_path', 's3_key'])
files_to_upload = []
if config['id'] == 'coquitlam':
# Upload unprocessed videos for Coquitlam.
download_dir = os.path.join(DOWNLOADS_DIR, config['id'])
for subdir in os.listdir(download_dir):
subdir = os.path.join(download_dir, subdir)
if not os.path.isdir(subdir):
continue
metadata_path = os.path.join(subdir, '_metadata.yaml')
meta = yaml_load(metadata_path)[0]
video_filename = os.path.basename(meta.url)
video_path = os.path.join(subdir, video_filename)
s3_base_path = '{}/{}/'.format(config['id'], meta.start_ts.split('-')[0])
if os.path.exists(video_path):
files_to_upload.extend([
FileUpload(metadata_path, s3_base_path + video_filename + '.metadata.yaml'),
FileUpload(video_path, s3_base_path + video_filename),
])
else:
for metadata_filename in sorted(filter(lambda f: f.endswith('.yaml'), os.listdir(VIDEOS_DIR))):
metadata_path = os.path.join(VIDEOS_DIR, metadata_filename)
prepped_video_info = yaml_load(metadata_path)
if prepped_video_info.config_id != config['id']:
continue
video_path = os.path.join(VIDEOS_DIR, prepped_video_info.video_filename)
if not os.path.exists(video_path):
print(video_path + " doesn't exist")
continue
s3_base_path = '{}/{}/'.format(config['id'], prepped_video_info.video_metadata.clip_start_utc.split('-')[0])
files_to_upload.extend([
FileUpload(metadata_path, s3_base_path + metadata_filename),
FileUpload(video_path, s3_base_path + prepped_video_info.video_filename),
])
client = S3Transfer(boto3.client('s3'))
bucket = config['s3_bucket']
for entry in files_to_upload:
print("Uploading {} to bucket {} at key {}".format(entry.src_path, bucket, entry.s3_key))
client.upload_file(entry.src_path, bucket, entry.s3_key)
if delete_after:
print("Deleting " + entry.src_path)
os.remove(entry.src_path)
def classinfo_from_modelinfo(mongodb, dbname, filepath):
modelinfo = common.yaml_load(filepath)
log.info("modelinfo: {}".format(modelinfo))
lbl_ids = modelinfo['classes']
classinfo = [{
'lbl_id': lbl_id,
'source': 'hmd',
'name': lbl_id
} for lbl_id in lbl_ids[1:]]
tblname = annonutils.get_tblname('CLASSINFO')
mongodb.connect(dbname)
mongodb.write(tblname, classinfo, idx_col='lbl_id')
def filter_data(df):
"""
Filtering data from errors and outliers
"""
df['tpep_pickup_datetime'] = df['tpep_pickup_datetime'].astype(
'datetime64')
df['tpep_dropoff_datetime'] = df['tpep_dropoff_datetime'].astype(
'datetime64')
row_len = len(df)
# Create duration column
df['trip_duration_min'] = (df['tpep_dropoff_datetime'] - df['tpep_pickup_datetime']) / \
np.timedelta64(1, 'm')
# Arround minute
df['trip_duration_min'] = df['trip_duration_min'].apply(round_min)
# Zero duration cleaning. Threshold < 1 minute
df.drop(df[df['trip_duration_min'] < 1].index, inplace=True, axis=0)
# Zero passenger cleaning. Threshold = 0 passenger
df.drop(df[df['passenger_count'] == 0].index, inplace=True)
# Zero distance cleaning. Threshold < 1 mile distance
df.drop(df[df['trip_distance'] < 1].index, inplace=True)
# only town region
town_square = common.yaml_load()['ny_square']
d_latitude, d_longitude = get_grid_params(grid_input=town_square,
grid_cells=50)
df['region'] = df.apply(get_region_5,
d_latitude=d_latitude,
d_longitude=d_longitude,
ny_grid=town_square,
grid_cells=50,
axis=1)
df.drop(df[df['region'] > 2500].index, inplace=True)
logging.debug(f"Has been filtered {row_len - len(df)} rows")
return df
def upload(config, delete_after):
yt_config = config['youtube']
client_creds = load_client_credentials()
tokens_file = tokens_file_for_id(config['id'])
if not os.path.isfile(tokens_file):
print("Need to perform 'youtube authorize' first.")
with open(tokens_file) as inf:
tokens = json.load(inf)
session = YouTubeSession(client_creds, tokens)
for metadata_filename in sorted(filter(lambda f: f.endswith('.yaml'), os.listdir(VIDEOS_DIR))):
metadata_path = os.path.join(VIDEOS_DIR, metadata_filename)
prepped_video_info = yaml_load(metadata_path)
if prepped_video_info.config_id != config['id']:
continue
video_path = os.path.join(VIDEOS_DIR, prepped_video_info.video_filename)
if not os.path.exists(video_path):
print(video_path + " doesn't exist")
continue
yt_video_res = build_youtube_resource(
prepped_video_info.title,
prepped_video_info.description,
pendulum.parse(prepped_video_info.video_metadata.start_ts),
coords=yt_config['location'],
location_desc=yt_config['location_desc'],
tags=yt_config['tags'],
privacy=yt_config['privacy'],
)
print(yt_video_res)
yt_video_id = session.upload(video_path, yt_video_res, yt_config['notify_subscribers'])
for playlist in prepped_video_info.playlists:
session.add_video_to_playlist(playlist, yt_video_id, yt_config['privacy'])
if delete_after:
for f in (metadata_path, video_path):
print("Deleting " + f)
os.remove(f)
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
# original lib
import common
########################################################################
########################################################################
# configuration and logs
########################################################################
logging.basicConfig(level=logging.DEBUG,
filename='logs.log',
format='%(asctime)s %(levelname)s:%(message)s')
path = common.yaml_load()['taxi_data']
########################################################################
def round_min(minute):
return round(minute, 2)
def get_grid_params(grid_input: dict, grid_cells: int):
d_latitude = (grid_input['latitude'][1] -
grid_input['latitude'][0]) / grid_cells
d_longitude = (grid_input['longitude'][1] -
grid_input['longitude'][0]) / grid_cells
return d_latitude, d_longitude
import librosa
import pandas as pd
from tqdm import tqdm
from scipy.io.wavfile import write
# original lib
import common
########################################################################
########################################################################
# configuration and logs
########################################################################
logging.basicConfig(level=logging.DEBUG,
filename='logs.log',
format='%(asctime)s %(levelname)s:%(message)s')
param = common.yaml_load()
iter_normal = ['normal', 'anomaly']
iter_type = {'audio': 'wav', 'vibro': 'txt', 'modbus': 'csv'}
########################################################################
def file_name_list(dir, dir_name, prefix_normal, prefix_type, ext):
"""
Return file name with extention in path
"""
file_name_list = sorted(
glob.glob(
"{dir}/{dir_name}/{prefix_normal}/{prefix_type}/*.{ext}".format(
dir=dir,
def load_date_metadata():
for dt in for_dates:
date_metadata_path = os.path.join(metadata_dir, dt.to_date_string() + '.yaml')
if not os.path.exists(date_metadata_path):
raise ValueError("No metadata downloaded for " + dt.to_date_string())
yield yaml_load(date_metadata_path)
########################################################################
########################################################################
# import additional python-library
########################################################################
import numpy
from sklearn import metrics
import common as com
from pytorch_model import VariationalAutoEncoder as VAE
import torch.utils.data
########################################################################
########################################################################
# load parameter.yaml
########################################################################
PARAM = com.yaml_load('./gan.yaml')
#######################################################################
########################################################################
# def
########################################################################
def save_csv(save_file_path, save_data):
with open(save_file_path, "w", newline="") as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(save_data)
def get_machine_id_list_for_test(target_dir, dir_name="test", ext="wav"):
"""
# from import
from tqdm import tqdm
from sklearn import metrics
try:
from sklearn.externals import joblib
except:
import joblib
# original lib
import common as com
import keras_model
########################################################################
########################################################################
# load parameter.yaml
########################################################################
param = com.yaml_load()
#######################################################################
########################################################################
# output csv file
########################################################################
def save_csv(save_file_path, save_data):
with open(save_file_path, "w", newline="") as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(save_data)
########################################################################
from vibro.vibro import vibro_write
from modbus.adapter import modbus_write
########################################################################
########################################################################
# import interface library
########################################################################
from tkinter import Tk, Button
from threading import Thread
from tkinter import messagebox as mb
from tkinter import filedialog as fd
from tkinter.filedialog import askopenfilename
########################################################################
#param loading
param = yaml_load("config.yaml")
audio_param = param['audio']
vibro_param = param['vibro']
modbus_param = param['modbus']
flagCOMPortRead = False
serialCOMPort = serial.Serial()
#main funcrions
def stop():
global flagCOMPortRead
flagCOMPortRead = False
print('Stop')
serialCOMPort.close()
writeThread.join()
########################################################################
# import additional python-library
########################################################################
import numpy
import common as com
import torch
import torch.utils.data
from torch import optim, nn
from torch.utils.data.dataset import Subset
from pytorch_model import AutoEncoder
########################################################################
# load parameter.yaml
########################################################################
param = com.yaml_load('./baseline.yaml')
########################################################################
########################################################################
# Dataset
########################################################################
class dcaseDataset(torch.utils.data.Dataset):
def __init__(self,
target_dir,
dir_name="train",
ext="wav",
n_mels=64,
frames=5,
n_fft=1024,
########################################################################
# import additional python-library
########################################################################
import numpy
from sklearn import metrics
import common as com
from pytorch_model import VariationalAutoEncoder as VAE
import torch.utils.data
########################################################################
########################################################################
# load parameter.yaml
########################################################################
PARAM = com.yaml_load('./vae.yaml')
#######################################################################
########################################################################
# def
########################################################################
def save_csv(save_file_path,
save_data):
with open(save_file_path, "w", newline="") as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(save_data)
def get_machine_id_list_for_test(target_dir,
dir_name="test",
sys.path.append('..')
import common as com
import pytorch_common
from model import *
########################################################################
########################################################################
# control stochastic behavior
########################################################################
deterministic_everything(2022, pytorch=True)
#######################################################################
########################################################################
# load parameter.yaml
########################################################################
params = EasyDict(com.yaml_load('config.yaml'))
#######################################################################
########################################################################
# def
########################################################################
def save_csv(save_file_path, save_data):
with open(save_file_path, "w", newline="") as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(save_data)
def get_machine_id_list_for_test(target_dir, dir_name="test", ext="wav"):
"""
sys.path.append('..')
import common as com
import pytorch_common
import model
########################################################################
########################################################################
# control stochastic behavior
########################################################################
deterministic_everything(2022, pytorch=True)
#######################################################################
########################################################################
# load parameter.yaml
########################################################################
param = com.yaml_load('config.yaml')
#######################################################################
########################################################################
# def
########################################################################
def save_csv(save_file_path, save_data):
with open(save_file_path, "w", newline="") as f:
writer = csv.writer(f, lineterminator='\n')
writer.writerows(save_data)
def get_machine_id_list_for_test(target_dir, dir_name="test", ext="wav"):
"""
