def get_video_from_playlist(playlistId): developer_key = getenv("DEVELOPER_KEY") log_level = getenv("LOG_LEVEL", "INFO") logger = get_logger("pyutils.youtube.get_video_from_playlist", log_level) youtube = build("youtube", "v3", developerKey=developer_key) playlistItems = [] pageToken = None while True: response = youtube.playlistItems().list(part="snippet", playlistId=playlistId, pageToken=pageToken, maxResults=100).execute() playlistItems += response["items"] logger.info("Fetch {} of {} results".format( len(playlistItems), response["pageInfo"]["totalResults"])) pageToken = response.get("nextPageToken", None) if pageToken == None: break return playlistItems
def import_directory(data_path, clean=False): log_level = getenv("LOG_LEVEL", "INFO") logger = get_logger("pyutils.mongo.import_directory", log_level) client = MongoClient(getenv("MONGO_URL")) database = client[getenv("MONGO_DB")] if clean: for collection in database.list_collection_names( include_system_collections=False): col = database[collection] col.drop() logger.warning("Drop collection {}".format(collection)) for filename in listdir(data_path): if filename.endswith(".json"): data = loads(open("{}/{}".format(data_path, filename), "r").read()) col = self.database[filename[:-5]] for record in data: try: self.logger.info("Insert collection {}".format( filename[:-5])) col.insert_one(record) except Exception as e: self.logger.error("Insert collection {} : {}".format( filename[:-5], str(e)))
"""Endpoints for indexing and retrieving documents""" from elasticsearch import TransportError from flask import Blueprint, request from commons import GenericException, build_response, get_logger from models import CheckinElastic, BusinessElastic, TipElastic, UserElastic, ReviewElastic from models.python import Tip, Checkin, User, Business, Review document = Blueprint('document', __name__) logger = get_logger(__name__) @document.route('', methods=['GET']) def get(name: str, doc_id: int): """Get a document by ID""" try: if name == 'businesses': resp = Business.hydrate(BusinessElastic.get(id=doc_id, index=name)) elif name == 'tips': resp = Tip.hydrate(TipElastic.get(id=doc_id, index=name)) elif name == 'users': resp = User.hydrate(UserElastic.get(id=doc_id, index=name)) elif name == 'checkins': resp = Checkin.hydrate(CheckinElastic.get(id=doc_id, index=name)) elif name == 'reviews': resp = Review.hydrate(ReviewElastic.get(id=doc_id, index=name)) else: raise GenericException('unknown index {}'.format(name), 400) except TransportError as e: raise GenericException(
from theconf.argument_parser import ConfigArgumentParser from torch.utils.data.dataset import Dataset, Subset from torchvision import transforms from torchvision.transforms.transforms import Compose, Resize from tqdm._tqdm import tqdm from theconf import Config as C from commons import tta_num, get_logger, decode, \ print_log, encoded_tta_default, mirror_expansion from conf import dataroot from imagenet import ImageNet from metrics import accuracy, Accumulator from networks import get_model from profile import Profiler logger = get_logger('learn2test') logger.setLevel(logging.DEBUG) if __name__ == '__main__': parser = ConfigArgumentParser(conflict_handler='resolve') parser.add_argument('--test-batch', type=int, default=32) parser.add_argument('--tta', type=str, default='center') parser.add_argument('--deform', type=str, default='') parser.add_argument('--corrupt', type=str, default='') args = parser.parse_args() assert args.dataset == 'imagenet' model_target = get_model(args.target_network, gpus=[0], num_classes=args.num_classes, train_aug=args.target_aug).eval() profiler = Profiler(model_target)
import os import pickle import json import logging import torchvision from tqdm._tqdm import tqdm from commons import get_logger from conf import dataroot _logger = get_logger('CachedDataLoader', logging.WARNING) class CachedDataLoader(object): NO_SEPARATE_EPOCH = 9999991958 CACHE_STATUS_UNKNOWN = 0 CACHE_STATUS_NO = -1 CACHE_STATUS_EXIST = 2 def __init__(self, loader, tag='cache', cache_max=5, step_max=10e10): self.epoch_cnt = -1 self.i = 0 self.step_max = step_max self.cache_max = cache_max self.loader = loader self.loader_iter = None self.tag = tag self.cache_status = CachedDataLoader.CACHE_STATUS_UNKNOWN
from tqdm._tqdm import tqdm from commons import tta_actions, tta_num, decode_desc, get_logger, \ mirror_expansion, pil_center_crop, decode from conf import dataroot from dataloader_cache import CachedDataLoader from imagenet import ImageNet from imagenet_c import corruptions from imagenet_c.corruptions import motion_blur from imagenet_c import corruption_tuple_tr, corruption_dict, corruption_tuple_vl from networks import get_model from remote_dataloader.loader import RemoteDataLoader from theconf import Config as C from conf import imagenet_path logger = get_logger('learn2test-data', logging.DEBUG) corruption_tuple = corruption_tuple_tr logger.info(corruption_tuple) def cutout(img, cutsize=16): if cutsize <= 0: return img w, h = img.size x0 = np.random.uniform(w) y0 = np.random.uniform(h) x1 = int(max(0, x0 - cutsize / 2.)) y1 = int(max(0, y0 - cutsize / 2.)) x2 = min(w, x0 + cutsize / 2.)
from dataloader import AugmentedDataset, get_dataset, CorruptedDataset from dataloader_cache import CachedDataLoader from architectures.efficientnet_pytorch.ema import EMA from architectures.efficientnet_pytorch.rmsproptf import RMSpropTF from metrics import Accumulator, prediction_correlation from networks import get_model from profile import Profiler from remote_dataloader.loader import RemoteDataLoader from theconf import Config as C from imagenet_c import corruption_dict from conf import sodeep_model from sodeep.sodeep import SpearmanLoss, load_sorter if __name__ == '__main__': logger = get_logger('learn2test-train') logger.setLevel(logging.DEBUG) parser = ConfigArgumentParser(conflict_handler='resolve') parser.add_argument('--tag', default='dev', type=str) parser.add_argument('--port', default=1958, type=int) parser.add_argument('--cv', default=0, type=int) parser.add_argument('--regularization', default=2, type=int) parser.add_argument('--ema-momentum', default=0.999, type=float) parser.add_argument('--data-cache', default=0, type=int) parser.add_argument('--cutout', default=-1, type=int) parser.add_argument('--aug-corrupt', default=1, type=int) parser.add_argument('--aug-p', default=0.4, type=float)