def get_video_from_playlist(playlistId):
developer_key = getenv("DEVELOPER_KEY")
log_level = getenv("LOG_LEVEL", "INFO")
logger = get_logger("pyutils.youtube.get_video_from_playlist", log_level)
youtube = build("youtube", "v3", developerKey=developer_key)
playlistItems = []
pageToken = None
while True:
response = youtube.playlistItems().list(part="snippet",
playlistId=playlistId,
pageToken=pageToken,
maxResults=100).execute()
playlistItems += response["items"]
logger.info("Fetch {} of {} results".format(
len(playlistItems), response["pageInfo"]["totalResults"]))
pageToken = response.get("nextPageToken", None)
if pageToken == None:
break
return playlistItems
def import_directory(data_path, clean=False):
log_level = getenv("LOG_LEVEL", "INFO")
logger = get_logger("pyutils.mongo.import_directory", log_level)
client = MongoClient(getenv("MONGO_URL"))
database = client[getenv("MONGO_DB")]
if clean:
for collection in database.list_collection_names(
include_system_collections=False):
col = database[collection]
col.drop()
logger.warning("Drop collection {}".format(collection))
for filename in listdir(data_path):
if filename.endswith(".json"):
data = loads(open("{}/{}".format(data_path, filename), "r").read())
col = self.database[filename[:-5]]
for record in data:
try:
self.logger.info("Insert collection {}".format(
filename[:-5]))
col.insert_one(record)
except Exception as e:
self.logger.error("Insert collection {} : {}".format(
filename[:-5], str(e)))
"""Endpoints for indexing and retrieving documents"""
from elasticsearch import TransportError
from flask import Blueprint, request
from commons import GenericException, build_response, get_logger
from models import CheckinElastic, BusinessElastic, TipElastic, UserElastic, ReviewElastic
from models.python import Tip, Checkin, User, Business, Review
document = Blueprint('document', __name__)
logger = get_logger(__name__)
@document.route('', methods=['GET'])
def get(name: str, doc_id: int):
"""Get a document by ID"""
try:
if name == 'businesses':
resp = Business.hydrate(BusinessElastic.get(id=doc_id, index=name))
elif name == 'tips':
resp = Tip.hydrate(TipElastic.get(id=doc_id, index=name))
elif name == 'users':
resp = User.hydrate(UserElastic.get(id=doc_id, index=name))
elif name == 'checkins':
resp = Checkin.hydrate(CheckinElastic.get(id=doc_id, index=name))
elif name == 'reviews':
resp = Review.hydrate(ReviewElastic.get(id=doc_id, index=name))
else:
raise GenericException('unknown index {}'.format(name), 400)
except TransportError as e:
raise GenericException(
from theconf.argument_parser import ConfigArgumentParser
from torch.utils.data.dataset import Dataset, Subset
from torchvision import transforms
from torchvision.transforms.transforms import Compose, Resize
from tqdm._tqdm import tqdm
from theconf import Config as C
from commons import tta_num, get_logger, decode, \
print_log, encoded_tta_default, mirror_expansion
from conf import dataroot
from imagenet import ImageNet
from metrics import accuracy, Accumulator
from networks import get_model
from profile import Profiler
logger = get_logger('learn2test')
logger.setLevel(logging.DEBUG)
if __name__ == '__main__':
parser = ConfigArgumentParser(conflict_handler='resolve')
parser.add_argument('--test-batch', type=int, default=32)
parser.add_argument('--tta', type=str, default='center')
parser.add_argument('--deform', type=str, default='')
parser.add_argument('--corrupt', type=str, default='')
args = parser.parse_args()
assert args.dataset == 'imagenet'
model_target = get_model(args.target_network, gpus=[0], num_classes=args.num_classes, train_aug=args.target_aug).eval()
profiler = Profiler(model_target)
import os
import pickle
import json
import logging
import torchvision
from tqdm._tqdm import tqdm
from commons import get_logger
from conf import dataroot
_logger = get_logger('CachedDataLoader', logging.WARNING)
class CachedDataLoader(object):
NO_SEPARATE_EPOCH = 9999991958
CACHE_STATUS_UNKNOWN = 0
CACHE_STATUS_NO = -1
CACHE_STATUS_EXIST = 2
def __init__(self, loader, tag='cache', cache_max=5, step_max=10e10):
self.epoch_cnt = -1
self.i = 0
self.step_max = step_max
self.cache_max = cache_max
self.loader = loader
self.loader_iter = None
self.tag = tag
self.cache_status = CachedDataLoader.CACHE_STATUS_UNKNOWN
from tqdm._tqdm import tqdm
from commons import tta_actions, tta_num, decode_desc, get_logger, \
mirror_expansion, pil_center_crop, decode
from conf import dataroot
from dataloader_cache import CachedDataLoader
from imagenet import ImageNet
from imagenet_c import corruptions
from imagenet_c.corruptions import motion_blur
from imagenet_c import corruption_tuple_tr, corruption_dict, corruption_tuple_vl
from networks import get_model
from remote_dataloader.loader import RemoteDataLoader
from theconf import Config as C
from conf import imagenet_path
logger = get_logger('learn2test-data', logging.DEBUG)
corruption_tuple = corruption_tuple_tr
logger.info(corruption_tuple)
def cutout(img, cutsize=16):
if cutsize <= 0:
return img
w, h = img.size
x0 = np.random.uniform(w)
y0 = np.random.uniform(h)
x1 = int(max(0, x0 - cutsize / 2.))
y1 = int(max(0, y0 - cutsize / 2.))
x2 = min(w, x0 + cutsize / 2.)
from dataloader import AugmentedDataset, get_dataset, CorruptedDataset
from dataloader_cache import CachedDataLoader
from architectures.efficientnet_pytorch.ema import EMA
from architectures.efficientnet_pytorch.rmsproptf import RMSpropTF
from metrics import Accumulator, prediction_correlation
from networks import get_model
from profile import Profiler
from remote_dataloader.loader import RemoteDataLoader
from theconf import Config as C
from imagenet_c import corruption_dict
from conf import sodeep_model
from sodeep.sodeep import SpearmanLoss, load_sorter
if __name__ == '__main__':
logger = get_logger('learn2test-train')
logger.setLevel(logging.DEBUG)
parser = ConfigArgumentParser(conflict_handler='resolve')
parser.add_argument('--tag', default='dev', type=str)
parser.add_argument('--port', default=1958, type=int)
parser.add_argument('--cv', default=0, type=int)
parser.add_argument('--regularization', default=2, type=int)
parser.add_argument('--ema-momentum', default=0.999, type=float)
parser.add_argument('--data-cache', default=0, type=int)
parser.add_argument('--cutout', default=-1, type=int)
parser.add_argument('--aug-corrupt', default=1, type=int)
parser.add_argument('--aug-p', default=0.4, type=float)