def __init__(self, model, config, criterion, optimizer, dataset, model_dir): """Initializes a BaseTrainer object Arguments: model {torch.nn.Module} -- PyTorch neural network model config {Configuration} -- Configuration object of settings. criterion {nn.loss} -- Loss function to optimize. optimizer {torch.optim} -- Optimizer object dataset {torch.utils.data.Dataset} -- A dataset object to use with dataloaders model_dir {str} -- Directory path to the model """ self.config = config self.model = model self.dataset = dataset self.criterion = criterion self.optimizer = optimizer self.loss = [] self.parameters = 3 self.save_path = os.path.join(model_dir, "states") self.tensorboard = utils.str_to_bool(self.config.tensorboard) self.cuda = utils.str_to_bool(self.config.cuda) self.gpu = str(self.config.gpu) self.verbose = utils.str_to_bool(self.config.verbose) self.validation = utils.str_to_bool(self.config.validation) self.clip = self.config.clip self.mode = self.config.mode self.batch_size = self.config.params.batch_size self.n_epochs = self.config.params.n_epochs self.train_fraction = self.config.params.train_fraction self.epochs = range(1, self.n_epochs+1) self.train_loader, self.val_loader = self._split_data(self.dataset, self.validation, self.train_fraction) # If logging with tensorboard, create a new log directory and remove if existing. if self.tensorboard: tensorboard_dir = os.path.join(model_dir, "logs/") if not os.path.exists(tensorboard_dir): os.makedirs(tensorboard_dir) else: try: shutil.rmtree(tensorboard_dir) except OSError as e: print ("Error: %s - %s." % (e.filename, e.strerror)) self.writer = SummaryWriter(tensorboard_dir)
def get_dnd(self): tree = self.xsi_get("services/donotdisturb") active, splash = [ utils.node_value(tree, x) for x in ("active", "ringSplash") ] return { "active": utils.str_to_bool(active), "ringSplash": utils.str_to_bool(splash) }
def event_csv_loader(filestr): with open(filestr, "r") as csvfile: mydict = {} reader = csv.reader(csvfile, delimiter=',') next(reader) for row in reader: mydict[row[0]] = { "event_type": row[2].strip(), "actor_type": row[1].strip(), "can_aoe": str_to_bool(row[3].strip()), "panic_blocked": str_to_bool(row[4].strip()) } return mydict
def get_call_forwarding(self, forward_type="always"): """Get call forwarding status for given forward type. Returns dict with keys 'active' (bool) and 'number' (str) """ tree = self.xsi_get("services/callforwarding%s" % forward_type) active, number = [ utils.node_value(tree, x) for x in ("active", "forwardToPhoneNumber") ] res = {"active": utils.str_to_bool(active), "number": number or ""} if forward_type == "always": splash = utils.node_value(tree, "ringSplash") res["ringSplash"] = utils.str_to_bool(splash) return res
def __init__(self): mishmash_config_reader = MishmashConfigReader(MishmashGrpcConfig.CONFIG_FILE_PATHS, MishmashGrpcConfig.ENV_VARIABLE_NAMES, MishmashGrpcConfig.NOT_REQUIRED_VARIABLE_NAMES) self.__raw_config = mishmash_config_reader.get_configuration() if not isinstance(self.__raw_config["MISHMASHIO_AUTH_METHOD"], str): raise MishmashWrongCredentialsException(f"please provide valid MISHMASHIO_AUTH_METHOD") self.__auth_method = self.__raw_config["MISHMASHIO_AUTH_METHOD"] self.__is_ssl = str_to_bool(self.__raw_config["MISHMASHIO_USE_SSL"]) if self.__is_ssl: if "MISHMASH_CERT_FILE_PATH" not in self.__raw_config: raise MishmashWrongCredentialsException(f"please provide valid MISHMASH_CERT_FILE_PATH") self.__trusted_certs = load_credential_from_file( self.__raw_config["MISHMASH_CERT_FILE_PATH"]) self.__server_address_list = parse_server_list(self.__raw_config["MISHMASHIO_SERVERS"], self.__is_ssl, MishmashGrpcConfig.DEFAULT_SSL_PORT, MishmashGrpcConfig.DEFAULT_PORT ) self.__auth_plugin = self.__get_authenticaiton_plugin()
def set_config(command, param): available_commands = { 'newline_on_ga': 'bool', 'health_sip_percentage': 'int', 'eat_lestagii': 'bool', 'basic_lestagii_health_percentage': 'int', 'combat_lestagii_health_percentage': 'int', 'basic_lestagii_mana_percentage': 'int', 'combat_lestagii_mana_percentage': 'int' } if command in available_commands.keys(): if available_commands[command] is 'bool': param = utils.str_to_bool(param) elif available_commands[command] is 'int': try: param = int(param) except ValueError: echo('Option "' + command + '" expects a numeric value') return False config[command] = param echo('Config option ' + command + ' updated') else: echo("Invalid configuration option")
def on_key_pressed(self, key): if str_to_bool(self.db.get('IS_HUNG_UP')): return { 'ok' : False, 'error' : "Phone is hung up" } # play dmtf tone; command = { "press" : self.map_key_to_tone(key) } mode = int(self.db.get('MODE')) logging.info("CURRENT MODE: %d" % mode) if mode == GATHER_MODE: logging.info("THIS IS A GATHER: %d" % key) try: gathered_keys = json.loads(self.db.get('gathered_keys')) print gathered_keys except Exception as e: logging.warning("could not get gathered_keys: ") print e, type(e) gathered_keys = [] gathered_keys += [key] self.db.set('gathered_keys', json.dumps(gathered_keys)) return self.send_command(command)
def index_vnf(fields, filename, final_path, new_version): """ Indexing function for VNFs """ user = get_user() fields['user'] = user fields['visibility'] = str_to_bool(request.form.get('visibility', 1)) data_ind = {'name': fields['name'], 'description': fields['description'], 'vendor': fields['vendor'], 'path': fields['path']} copyfile(filename, final_path + '/' + fields.get('id') + "-" + fields.get('version') + '.tar.gz') yaml.dump(fields, open(final_path + '/' + 'metadata.yaml', 'w')) index = yaml.load(open('/repository/index.yaml'), Loader=yaml.FullLoader) if new_version: index['vnf_packages'][fields.get('id')][fields.get('version')] = data_ind if version.parse(index['vnf_packages'][fields.get('id')]['latest']) < version.parse( fields.get('version')): index['vnf_packages'][fields.get('id')]['latest'] = fields.get('version') else: index['vnf_packages'][fields.get('id')] = {fields.get('version'): data_ind} index['vnf_packages'][fields.get('id')]['latest'] = fields.get('version') if not fields['visibility']: private_packages = dbclient['private_packages']['vnf'] private_packages.insert_one({'id': fields.get('id'), 'user': user}) dependencies = dbclient['dependencies']['vnf'] dependencies.insert_one({'id': fields.get('id'), 'images': fields.get('images')}) yaml.dump(index, open('/repository/' + 'index.yaml', 'w'))
def get(self): custom_filter = SaleList.parser.parse_args() response = [] if custom_filter.get('deleted'): sales = SaleModel.filter_by_deleted(str_to_bool(custom_filter.get('deleted'))) else: sales = SaleModel.find_all() for sale in sales: products_from_sale = SoldProductModel.find_by_sale_id(sale.sale_id) # payments_from_sale = TransactionModel.find_by_sale_id(sale.sale_id) sale_element = {"sale_id": sale.sale_id, "date": str(sale.date)[:19], "client": sale.client.json(), "total": sale.total, "discount": sale.discount, "user_commission": sale.user_commission, "promoter_commission": sale.promoter_commission, "deleted": sale.deleted, # "payments": [payment.json() for payment in payments_from_sale], "products": [product.json() for product in products_from_sale]} if sale.promoter: sale_element["promoter"] = sale.promoter.json() if sale.user: sale_element["seller"] = sale.user.json() response.append(sale_element) return response
def test_str_to_bool(): assert str_to_bool('true') == True assert str_to_bool('false') == False assert str_to_bool('TRUE') == True assert str_to_bool('FALSE') == False assert str_to_bool('FAIL') is None assert str_to_bool('Fail', True) == True assert str_to_bool(None, True) == True assert str_to_bool('Fail', 'Oh!') == 'Oh!'
def get(self): custom_filter = TransactionList.parser.parse_args() if custom_filter.get('deleted'): return [transaction.json() for transaction in TransactionModel.filter_by_deleted(str_to_bool(custom_filter.get('deleted')))] else: return [transaction.json() for transaction in TransactionModel.find_all()]
def ResolveVarToBool(self, in_var, default=False): retVal = default try: resolved_var = self.ResolveVarToStr(in_var) retVal = utils.str_to_bool(resolved_var, default) except: pass return retVal
def get_remote_office(self): """Get Remote Office call setting """ tree = self.xsi_get("services/remoteoffice") active, number = [ utils.node_value(tree, x) for x in ("active", "remoteOfficeNumber") ] return {"active": utils.str_to_bool(active), "number": number or ""}
def on_pick_up(self): logging.info("picking up") self.reset_for_call() p = Process(target=self.run_script) p.start() return { 'ok' : not str_to_bool(self.db.get('IS_HUNG_UP')) }
def get_broadworks_anywhere(self): """Get BroadWorks Anywhere call setting. """ tree = self.xsi_get("services/broadworksanywhere") dial, paging = [ utils.node_value(tree, x) for x in ("alertAllLocationsForClickToDialCalls", "alertAllLocationsForGroupPagingCalls") ] return { "alertAllLocationsForClickToDialCalls": utils.str_to_bool(dial), "alertAllLocationsForGroupPagingCalls": utils.str_to_bool(paging), "locations": map(lambda elem: utils.node_list(elem), tree.findall(utils.ns_escape("locations/location"))) }
def __init__(self, **kwargs): """ Constructs a Fabric object """ # implementations must respect this self.partial_cables = utils.str_to_bool( kwargs.get('partial_cables', '0')) self._routers = {} # radix->[router, count] self._cables = {} # actual_length->[cable,count] self._cable_granularity = utils.meters( kwargs.get('cable_granularity', '0.5m'))
def set_options_config(self, options): self.BACKLOG_ENABLED = str_to_bool(options["BACKLOG_ENABLED"]) self.WEEKLY_ARCHIVE = str_to_bool(options["WEEKLY_ARCHIVE"]) self.DAILY_RESET = str_to_bool(options["DAILY_RESET"]) self.UPDATE_DURATION = str_to_bool(options["UPDATE_DURATION"]) self.WEEKLY_BURN_CHART = str_to_bool(options["WEEKLY_BURN_CHART"]) self.STORY_POINTS_ENABLED = str_to_bool( options["STORY_POINTS_ENABLED"]) self.BURN_CHART_TYPE = options["BURN_CHART_TYPE"]
def get_broadworks_mobility(self): """Get BroadWorks Mobility call setting. Returns active state and primary mobile number only. """ tree = self.xsi_get("services/broadworksmobility") active = utils.node_value(tree, "active") # xpath looks more complex because can't specify default # namespece (without prefix) in xpath function arguments number = tree.xpath("/*[name()='BroadWorksMobility']" "/*[name()='mobileIdentity']" "/*[name()='primary'][text() = 'true']" "/parent::*" "/*[name()='mobileNumber']/text()") phone = number[0] if number else "" return {"active": utils.str_to_bool(active), "mobileNumber": phone}
def get_simultaneous_ring(self, **kwargs): """Get Simultaneous Ring call setting. """ tree = self.xsi_get("services/simultaneousringpersonal") active, ring = [ utils.node_value(tree, x) for x in ("active", "incomingCalls") ] return { "active": utils.str_to_bool(active), "incomingCalls": (ring == "Do not Ring if on a Call"), "simRingLocations": map( lambda elem: utils.node_list(elem), tree.findall( utils.ns_escape("simRingLocations/simRingLocation"))) }
def on_hang_up(self): logging.info("hanging up") self.send_command({ 'stop_audio' : True }) stop_daemon(self.conf['d_files']['module']) self.db.set('IS_HUNG_UP', True) try: current_record_pid = self.db.get("CURRENT_RECORD_PID") if current_record_pid is not None: logging.debug("current_record_pid: %s" % current_record_pid) os.kill(int(current_record_pid), signal.SIGKILL) self.db.delete("CURRENT_RECORD_PID") else: logging.debug("not currently recording") except Exception as e: logging.warning("NO CURRENT RECORD TO KILL") print e, type(e) return { 'ok' : str_to_bool(self.db.get('IS_HUNG_UP')) }
def delete_nsd(nsdId): """ Network Service Delete on OSM and repository """ try: logger.info(request) all = str_to_bool(request.form.get('all', 0)) user = get_user() index = yaml.load(open('/repository/index.yaml'), Loader=yaml.FullLoader) ns = index.get('ns_packages').get(nsdId, {}) if not len(ns): raise Exception('Network Service not found') latest = ns['latest'] ns_metadata = yaml.load(open('/repository/ns/' + nsdId + '/' + latest + '/metadata.yaml'), Loader=yaml.FullLoader) if user != ns_metadata['user']: raise Exception('User {} have not uploaded this Network Service. This user has not permissions for deleting' ' this'.format(user)) ns_id = dbclient['onboarded']['ns'].find_one({'ns': nsdId}) if ns_id: nbiUtil.delete_nsd(ns_id.get('nsid')) logger.info('NS {} deleted in OSM'.format(nsdId)) if all or len(ns) == 2: del index['ns_packages'][nsdId] rmtree('/repository/ns/' + nsdId) else: del index['ns_packages'][nsdId][latest] rmtree('/repository/ns/' + nsdId + '/' + latest) yaml.dump(index, open('/repository/' + 'index.yaml', 'w')) dbclient['onboarded']['ns'].delete_one({'ns': nsdId}) dbclient['private_packages']['nsd'].delete_one({'id': nsdId}) return jsonify({}), 204 except Exception as e: return jsonify({"detail": str(e), "code": type(e).__name__, "status": 400}), 400
def op_size(self, op, is_right_click=False): """ 更改分辨率和屏幕密度操作 """ serial = self.serial size_str = '' density_str = '' # 还原 if op == OP.SIZE_ORIGINAL: size_arr = util_atool.get_resolution(serial) density_arr = util_atool.get_density(serial) size_str = size_arr[0] density_str = density_arr[0] # 右键只进行查询 if is_right_click: s = "分辨率信息为:\n宽x高:{0},DPI:{1}".format(size_str, density_str) s += "\nsize:{0},density:{1}".format(size_arr, density_arr) utils.showinfo(s) return else: s = "分辨率 还原,size:{0}\tdensity:{1}".format( size_arr, density_arr) print(s) self.msg(s) # 等比缩放 elif op == OP.SIZE_SCALE: scale = 3 if is_right_click else 2 size_arr = util_atool.get_resolution(serial) s = size_arr[0] arr = s.split("x") w = int(arr[0]) h = int(arr[1]) # density_str = density_arr[0] size_str = "{0}x{1}".format(w * scale, h * scale) ps = "分辨率 等比缩放{0}倍, 仅更改宽x高".format(scale) ps += "\nsize:{0}\t{1}".format(size_arr, size_str) print(ps) self.msg(ps) # 自定义 elif op == OP.SIZE_CUSTOM: jf = setting.read_setting() if utils.str_to_bool(jf["size_on"]): size_str = jf["size"] if utils.str_to_bool(jf["density_on"]): density_str = jf['density'] if is_right_click: arr = [ "分辨率 自定义配置信息:", "\n宽x高:{0}, 是否启用:{1}".format(jf["size"], jf["size_on"]), "\nDPI:{0}, 是否启用:{1}".format(jf["density"], jf["density_on"]), ] utils.showinfo("".join(arr)) return else: s = "设置为 自定义分辨率,size:{0},density:{1}".format( size_str, density_str) print(s) self.msg(s) if size_str: self.th(util_atool.set_resolution, (serial, size_str, density_str))
# !/usr/bin/python2 """ DATABASE """ __all__ = ['database'] # Import required python libraries import os, sys import sqlite3 from utils import sprint, eprint, str_to_bool count = 0 folder = '/var/sqlite' file = os.getenv('DB_FILE') path = os.path.join(folder, file) in_memory = str_to_bool(os.getenv('DB_MEMORY')) if in_memory: database = ":memory:" else: database = path save_on_exit = str_to_bool(os.getenv('DB_SAVE')) # Init database if not os.path.exists(folder): os.makedirs(folder) try: with sqlite3.connect(database) as sql: sql.execute( "CREATE TABLE IF NOT EXISTS queue(ip TEXT PRIMARY KEY, lock BOOL DEFAULT 0, hits INTEGER DEFAULT 0, date DATETIME);" ) sql.execute( "CREATE TABLE IF NOT EXISTS auth(ip TEXT PRIMARY KEY, snmp_version TEXT, snmp_community TEXT, snmp_aprot TEXT, snmp_apass TEXT, snmp_seng_id TEXT, snmp_ceng_id TEXT, snmp_level TEXT, snmp_context TEXT, snmp_user TEXT, snmp_pprot TEXT, snmp_ppass TEXT, snmp_boots TEXT);"
def __init__(self, **kwargs): super(Hyperx2DStripe, self).__init__(**kwargs) # mandatory self._concentration = None self._widths = None self._weights = None self._chassis = None # chassis per rack # optional self._rack_stripe = False # parse kwargs for key in kwargs: if key == 'concentration': assert self._concentration == None, 'duplicate concentration' self._concentration = int(kwargs[key]) elif key == 'widths': assert self._widths == None, 'duplicate widths' self._widths = utils.str_to_int_list(kwargs[key]) elif key == 'weights': assert self._weights == None, 'duplicate weights' self._weights = utils.str_to_int_list(kwargs[key]) elif key == 'chassis': assert self._chassis == None, 'duplicate chassis' self._chassis = int(kwargs[key]) elif key == 'rack_stripe': self._rack_stripe = utils.str_to_bool(kwargs[key]) elif key in super(Hyperx2DStripe, self).using_options(): pass else: assert False, 'unknown option key: {}'.format(key) # check mandatory were given assert (self._concentration != None and self._widths != None and self._weights != None and self._chassis != None), \ 'concentration, widths, weights, and chassis must all be specified' # check sizes if len(self._widths) != len(self._weights) and len(self._widths) > 0: raise ValueError('HyperX widths and weight must be equal length') assert len( self._widths) <= 2, 'This topology supports up to 2 dimensions' for width in self._widths: assert width >= 2, 'HyperX widths must be > 1' # compute number of routers and nodes self._routers = functools.reduce(operator.mul, self._widths) self._nodes = self._routers * self._concentration # adjust the topology into a 2D topology even if it isn't dims = len(self._widths) for more_dims in range(dims, 2): self._widths.append(1) self._weights.append(0) assert len(self._widths) == 2 # determine total number of racks self._racks = math.ceil( (self._widths[0] * self._widths[1]) / self._chassis) # lengths self._lenmax = 0 self._lenmin = 9999999 self._lensum = 0 self._cblcnt = 0
# !/usr/bin/python2 """ DEVICE CONFIGURATION FILES """ __all__ = ['options_snmp', 'options_config', 'options_database', 'options_threads'] # Import required python libraries import os, sys import time from shutil import copyfile from subprocess import Popen, PIPE from stat import ST_MTIME from utils import eprint, sprint, str_to_bool, ConfigFileError, DiffProcedureError compare = str_to_bool(os.getenv('CONFIG_COMPARE')) """ generate diff file for configurations in one folder """ no_duplicates = str_to_bool(os.getenv('CONFIG_NODUP')) """ delete new configurations without changes """ folder = os.getenv('CONFIG_FOLDER') """ folder to store device configurations in 'format' macro, fields syslog, l0ip """ file = os.getenv('CONFIG_FILE') """ device configuration names in 'format' macro, fields syslog, l0ip, n """ diff = os.getenv('CONFIG_DIFF') """ device configuration different file name in 'format' macro, fields syslog, l0ip """ _diff_cmp = ['/usr/bin/diff', '-qs'] _diff_out = ['/usr/bin/diff', os.getenv('CONFIG_DIFF_OPT')] def do_save(src, msg, opt): l0ip = get_l0ip(msg['SOURCEIP']) dst_folder = get_config_folder(msg, l0ip=l0ip) if not os.path.exists(dst_folder): os.makedirs(dst_folder)
def main(*args): # load stop words stop_words = get_stop_words() plot = const.PLOT_DEFAULT print_ = const.PRINT_DEFAULT max_features = None random_state = const.RANDOM_STATE_DEFAULT order = -1 # default descending order wordcloud_n = None wordcloud_ = False cos_sim = False even_distrib = const.EVEN_DISTRIB_DEFAULT plt.rcParams.update({'font.size': const.FONT_SIZE_DEFAULT}) pre_vec = False limit_size = False min_df = 1 max_df = 1.0 param_compare = False # print command line arguments for arg in args: k = arg.split("=")[0] v = arg.split("=")[1] if k == 'plot': plot = utils.str_to_bool(v) elif k == 'print': print_ = utils.str_to_bool(v) elif k == 'max_features': max_features = int(v) elif k == 'stop_words': if utils.str_to_bool(v) == False: stop_words = None elif k == 'random_state': random_state = int(v) elif k == 'order': order = int(v) elif k == 'wordcloud': wordcloud_ = utils.str_to_bool(v) elif k == 'wordcloud_n': wordcloud_n = int(v) elif k == 'cos_sim': cos_sim = utils.str_to_bool(v) elif k == 'font_size': plt.rcParams.update({'font.size': int(v)}) elif k == 'even_distrib': even_distrib = utils.str_to_bool(v) elif k == 'pre_vec': pre_vec = utils.str_to_bool(v) elif k == 'limit_size': limit_size = utils.str_to_bool(v) elif k == 'min_df': min_df = int(v) elif k == 'max_df': max_df = float(v) if max_df > 1: max_df = int(max_df) elif k == 'param_compare': param_compare = utils.str_to_bool(v) else: print("Unknown param: {}".format(k)) if print_: print() print("-- Analysis config --") print("even_distrib: {}".format(even_distrib)) print("stop_words: {}".format(stop_words != None)) print("max_features: {}".format(max_features)) print("random_state: {}".format(random_state)) print("wordcloud: {}".format(wordcloud_)) print("wordcloud_n: {}".format(wordcloud_n)) print("order: {}".format(order)) print("cos_sim: {}".format(cos_sim)) print("param_compare: {}".format(param_compare)) print("pre_vec: {}".format(pre_vec)) print("limit_size: {}".format(limit_size)) print("min_df: {}".format(min_df)) print("max_df: {}".format(max_df)) print("plot: {}".format(plot)) print("--------------------") print() gen_spotify_df = pd.read_csv(const.GEN_SPOTIFY) clean_spotify_df = pd.read_csv(const.CLEAN_SPOTIFY) if even_distrib == False: clean_spotify_df = pd.read_csv(const.CLEAN_UNEVEN_SPOTIFY) gen_deezer_df = pd.read_csv(const.GEN_DEEZER) clean_deezer_df = pd.read_csv(const.CLEAN_DEEZER) if even_distrib == False: clean_deezer_df = pd.read_csv(const.CLEAN_UNEVEN_DEEZER) datasets = [ (const.SPOTIFY, clean_spotify_df), (const.DEEZER, clean_deezer_df), ] vectorizer = CountVectorizer( stop_words=stop_words, ngram_range=(1, 1), min_df=min_df, max_df=max_df, max_features=max_features, binary=True, ) # word clouds if wordcloud_: top_n = gen_word_cloud_grid( const.SPOTIFY, clean_spotify_df, vectorizer=vectorizer, n=wordcloud_n, order=order, random_state=random_state, print_=print_ ) spotify_shared, spotify_unique = get_shared_words(top_n) top_n = gen_word_cloud_grid( const.DEEZER, clean_deezer_df, vectorizer=vectorizer, n=wordcloud_n, order=order, random_state=random_state, print_=print_ ) deezer_shared, deezer_unique = get_shared_words(top_n) if print_: print() print("Spotify: count shared={}".format( len(spotify_shared)/len(spotify_unique))) print("Deezer: count shared={}".format( len(deezer_shared)/len(deezer_unique))) print() # cosine similarity if cos_sim: for name, dataset in datasets: if pre_vec: dataset = utils.get_vectorized_df(dataset, vectorizer) print("{} class data similarity analysis...".format(name)) for i in dataset.y.unique(): class_df = utils.get_class_based_data( dataset, i, random_state=random_state, include_other_classes=True, even_distrib=False, limit_size=limit_size, print_=True, ) if pre_vec == False: class_df = utils.get_vectorized_df(class_df, vectorizer) pos_df = utils.get_class_based_data(class_df, 1) pos_df.pop('y') ave_pos = utils.get_average_cos_sim(pos_df.values) neg_df = utils.get_class_based_data(class_df, -1.0) neg_df.pop('y') ave_neg = utils.get_average_cos_sim(neg_df.values) ave_between = utils.get_average_cos_sim( pos_df.values, neg_df.values) print("class {}".format(i)) print("data shape: {}".format(class_df.shape)) print("average positive cosine similarity: {}".format(ave_pos)) print("average negative cosine similarity: {}".format(ave_neg)) print("average between cosine similarity: {}".format(ave_between)) print("(pos - between )+ (neg - between) percentage = {} ".format( (ave_pos - ave_between) / ave_pos + (ave_neg - ave_between) / ave_neg )) print() if param_compare: # min_df vs pos_sim, neg_sim, between_sim params_grid = { 'min_df': [i for i in range(1, 15)], 'max_df': np.arange(0.1, 1.0, 0.1), } for name, dataset in datasets: for i in dataset.y.unique(): df = utils.get_class_based_data( dataset, i, random_state=random_state, include_other_classes=True, even_distrib=False, limit_size=limit_size, ) for p, v in params_grid.items(): print("Comparing cosine similarity vs {} for {} Class {} data...".format(p, name, i)) vectorizer = CountVectorizer( stop_words=stop_words, ngram_range=(1, 1), min_df=min_df, max_df=max_df, max_features=max_features, binary=True, ) pos_sim = [] neg_sim = [] between_sim = [] diff = [] for j in range(len(v)): vectorizer.set_params(**{p: v[j]}) class_df = utils.get_vectorized_df(df, vectorizer) pos_df = utils.get_class_based_data(class_df, 1) pos_df.pop('y') ave_pos = utils.get_average_cos_sim(pos_df.values) neg_df = utils.get_class_based_data(class_df, -1.0) neg_df.pop('y') ave_neg = utils.get_average_cos_sim(neg_df.values) ave_between = utils.get_average_cos_sim( pos_df.values, neg_df.values) pos_sim.append(ave_pos) neg_sim.append(ave_neg) between_sim.append(ave_between) diff.append((ave_pos - ave_between)/ave_pos + (ave_neg - ave_between)/ave_neg) plt.figure() plt.title("{} Class {}: {} vs cosine similarity".format(name,i, p)) pos_sim = np.array(list(zip(v, pos_sim))) neg_sim = np.array(list(zip(v, neg_sim))) between_sim = np.array(list(zip(v, between_sim))) diff = np.array(list(zip(v, diff))) plt.plot(pos_sim[:, 0], pos_sim[:, 1], label='pos sim') plt.plot(neg_sim[:, 0], neg_sim[:, 1], label='neg sim') plt.plot(between_sim[:, 0], between_sim[:, 1], label='between sim') plt.plot(diff[:, 0], diff[:, 1], label='sim difference (%)') plt.xlabel(p) plt.legend() # grid search eval if plot: plt.draw() plt.show()
def get_gpio_status(self): # maybe this will be something more substantial... return str_to_bool(self.db.get('GPIO_STATUS'))
def cli_main(monitor: str, stat: str, download: str, dir: str): monitor = utils.str_to_bool(monitor) download = utils.str_to_bool(download) assert stat in ['ens', 'mean', 'sprd'], 'stat must be ens, mean, or sprd' retr = GEFSRetrieve(download_dir=dir, download=download, monitor=monitor) retr.run(stat)
def load_aliases_from_xml(file): xmlfile = parse(file) group = xmlfile.getroot() new_group = {} # Ensure these triggers have a group if group.tag != 'group': print "load_aliases_from_xml: File %s not contained in a <group>" % file return False # Gracefully handle group name being omitted try: new_group['name'] = group.attrib['name'] except KeyError: print "load_aliases_from_xml: Group name not defined in %s" % file return False # Optionally apply group enabled/disabled try: new_group['enabled'] = group.attrib['enabled'] new_group['enabled'] = str_to_bool(new_group['enabled']) except KeyError: new_group['enabled'] = False core.add_alias_group(new_group['name'], new_group['enabled']) # Load triggers for group for alias in group: new_alias = { 'name': None, 'type': None, 'pattern': None, 'enabled': True, 'function': [], 'send': [] } for node in alias: if node.tag == 'name': new_alias['name'] = node.text if node.tag == 'type': new_alias['type'] = node.text if node.tag == 'pattern': new_alias['pattern'] = node.text if node.tag == 'enabled': new_alias['enabled'] = str_to_bool(node.text) if node.tag == 'function': new_alias['function'].append(node.text) if node.tag == 'send': new_alias['send'].append(node.text) # Validate <trigger> keys = new_alias.keys() functions = [] # There has to be a better way..... if 'name' not in keys or 'type' not in keys or 'pattern' not in keys: print "load_alias_from_xml: Required alias element missing in %s" % file return False # Validate trigger types if new_alias['type'] not in ('exact', 'substring', 'regex', 'startswith', 'endswith'): print "load_alias_from_xml: Invalid alias type in %s" % file return False # Figure out enabled try: if new_alias['enabled'] is False: pass except KeyError: new_alias['enabled'] = False if len(new_alias['send']) > 0: for send in new_alias['send']: functions.append('send ' + send) if len(new_alias['function']) > 0: for function in new_alias['function']: functions.append(function) if new_alias['function'] is not None: core.add_alias( new_alias['name'], new_group['name'], new_alias['type'], new_alias['pattern'], functions, new_alias['enabled'], ) else: print "load_alias_from_xml: No function defined for generic trigger in %s" % file return False # Done! return new_group['name']
import os from utils import str_to_bool LR = 0.001 EPOCHS = os.getenv("EPOCHS", 5) _BATCHSIZE = 64 R_MEAN = 123.68 G_MEAN = 116.78 B_MEAN = 103.94 BUFFER = 256 DEFAULT_IMAGE_SIZE = 224 NUM_CHANNELS = 3 NUM_CLASSES = 1001 NUM_IMAGES = {"train": 1_281_167, "validation": 50000} NUM_TRAIN_FILES = 1024 SHUFFLE_BUFFER = 100 DATA_LENGTH = int( os.getenv("FAKE_DATA_LENGTH", 1_281_167) ) # How much fake data to simulate, default to size of imagenet dataset DATASET_NAME = "ImageNet" DISTRIBUTED = str_to_bool(os.getenv("DISTRIBUTED", "False"))
def main(*args): save = False data_size = -1 spotify_thresh = 0 deezer_thresh = 0 class_per_quad = 1 dist_mode = 'furthest_from_mean' for arg in args: k = arg.split("=")[0] v = arg.split("=")[1] if k == 'save': save = str_to_bool(v) elif k == 'data_size': data_size = int(v) elif k == 'spotify_thresh': spotify_thresh = float(v) elif k == 'deezer_thresh': deezer_thresh = float(v) elif k == 'class_per_quad': class_per_quad = int(v) elif k == 'dist_mode': dist_mode = str(v) print() print("--- Clean config ---") print("save: {}".format(save)) print("data_size: {}".format(data_size)) print("spotify_thresh: {}".format(spotify_thresh)) print("deezer_thresh: {}".format(deezer_thresh)) print("class_per_quad: {}".format(class_per_quad)) print("dist_mode: {}".format(dist_mode)) print("--------------------") print() # SPOTIFY DATASET cleaned_df = clean(const.GEN_SPOTIFY) df = gen_labels( cleaned_df, cross_over_val=0.5, thresh=spotify_thresh, data_size=data_size, class_per_quadrant=class_per_quad, dist_mode=dist_mode, ) check_dup(df, "Error: Duplicates!!!!!!!!!!!!!!!!!") if save: # save as csv (don't include indices in .csv) if data_size > 0 and dist_mode != 'furthest_from_mean': print("Saving result to {}...\n".format(const.CLEAN_SPOTIFY)) df.to_csv(const.CLEAN_SPOTIFY, index=False) else: print("Saving result to {}...\n".format( const.CLEAN_UNEVEN_SPOTIFY)) df.to_csv(const.CLEAN_UNEVEN_SPOTIFY, index=False) # DEEZER DATASET cleaned_df = clean( const.GEN_DEEZER, # word_count_range=(30, 600), # unique_words_range=(10, 300) ) df = gen_labels( cleaned_df, cross_over_val=0, thresh=deezer_thresh, data_size=data_size, class_per_quadrant=class_per_quad, dist_mode=dist_mode, ) check_dup(df, "Error: Duplicates!!!!!!!!!!!!!!!!!") if save: # save as csv (don't include indices in .csv) if data_size > 0 and dist_mode != 'furthest_from_mean': print("Saving result to {}...".format(const.CLEAN_DEEZER)) df.to_csv(const.CLEAN_DEEZER, index=False) else: print("Saving result to {}...".format(const.CLEAN_UNEVEN_DEEZER)) df.to_csv(const.CLEAN_UNEVEN_DEEZER, index=False)
def load_triggers_from_xml(file): xmlfile = parse(file) group = xmlfile.getroot() new_group = {} # Ensure these triggers have a group if group.tag != 'group': print "load_triggers_from_xml: File %s not contained in a <group>" % file return False # Gracefully handle group name being omitted try: new_group['name'] = group.attrib['name'] except KeyError: print "load_triggers_from_xml: Group name not defined in %s" % file return False # Optionally apply group enabled/disabled try: new_group['enabled'] = group.attrib['enabled'] new_group['enabled'] = str_to_bool(new_group['enabled']) except KeyError: new_group['enabled'] = False core.add_trigger_group(new_group['name'], new_group['enabled']) # Load triggers for group for trigger in group: new_trigger = { 'name': None, 'type': None, 'pattern': None, 'enabled': True, 'function': [], 'pfunction': [], 'pdisable_trigger': [], 'affliction': [], 'cure': [], 'bcure': [], 'symptom': [], 'gag': [], 'enable_group': [], 'disable_group': [], 'enable_trigger': [], 'disable_trigger': [], 'ondef': [], 'undef': [], 'send': [], 'deflist': None } for node in trigger: if node.tag == 'name': new_trigger['name'] = node.text elif node.tag == 'type': new_trigger['type'] = node.text elif node.tag == 'pattern': new_trigger['pattern'] = node.text elif node.tag == 'enabled': new_trigger['enabled'] = str_to_bool(node.text) elif node.tag == 'function': new_trigger['function'].append(node.text) elif node.tag == 'pfunction': new_trigger['pfunction'].append(node.text) elif node.tag == 'pdisable_trigger': new_trigger['pdisable_trigger'].append(node.text) elif node.tag == 'affliction': new_trigger['affliction'].append(node.text) elif node.tag == 'symptom': new_trigger['symptom'].append(node.text) elif node.tag == 'cure': new_trigger['cure'].append(node.text) elif node.tag == 'bcure': new_trigger['bcure'].append(node.text) elif node.tag == 'gag': new_trigger['gag'].append(node.text) elif node.tag == 'send': new_trigger['send'].append(node.text) elif node.tag == 'enable_group': new_trigger['enable_group'].append(node.text) elif node.tag == 'disable_group': new_trigger['disable_group'].append(node.text) elif node.tag == 'enable_trigger': new_trigger['enable_trigger'].append(node.text) elif node.tag == 'disable_trigger': new_trigger['disable_trigger'].append(node.text) elif node.tag == 'undef': new_trigger['undef'].append(node.text) elif node.tag == 'ondef': new_trigger['ondef'].append(node.text) elif node.tag == 'deflist': new_trigger['deflist'] = node.text # Validate <trigger> keys = new_trigger.keys() # There has to be a better way..... if 'name' not in keys or 'type' not in keys or 'pattern' not in keys: print "load_triggers_from_xml: Required trigger element missing in %s" % file return False # Validate trigger types if new_trigger['type'] not in ('exact', 'substring', 'regex', 'startswith', 'endswith'): print "load_triggers_from_xml: Invalid trigger type in %s" % file return False # Figure out enabled try: if new_trigger['enabled'] is False: pass except KeyError: new_trigger['enabled'] = False functions = [] if len(new_trigger['affliction']) > 0: for affliction in new_trigger['affliction']: functions.append('affliction ' + affliction) if len(new_trigger['symptom']) > 0: for symptom in new_trigger['symptom']: functions.append('symptom ' + symptom) if len(new_trigger['cure']) > 0: for cure in new_trigger['cure']: functions.append('cure ' + cure) if len(new_trigger['bcure']) > 0: for bcure in new_trigger['bcure']: functions.append('bcure ' + bcure) if len(new_trigger['gag']) > 0: for gag in new_trigger['gag']: functions.append('gag ' + gag) if len(new_trigger['send']) > 0: for send in new_trigger['send']: functions.append('send ' + send) if len(new_trigger['enable_group']) > 0: for group in new_trigger['enable_group']: functions.append('enable_group ' + group) if len(new_trigger['disable_group']) > 0: for group in new_trigger['disable_group']: functions.append('disable_group ' + group) if len(new_trigger['enable_trigger']) > 0: for trigger in new_trigger['enable_trigger']: functions.append('enable_trigger ' + trigger) if len(new_trigger['disable_trigger']) > 0: for trigger in new_trigger['disable_trigger']: functions.append('disable_trigger ' + trigger) if len(new_trigger['ondef']) > 0: for defence in new_trigger['ondef']: functions.append('ondef ' + defence) if len(new_trigger['undef']) > 0: for defence in new_trigger['undef']: functions.append('undef ' + defence) if new_trigger['deflist'] is not None: functions.append('deflist ' + new_trigger['deflist']) if len(new_trigger['function']) > 0: for function in new_trigger['function']: functions.append(function) if len(new_trigger['pfunction']) > 0: for function in new_trigger['pfunction']: functions.append('pfunction ' + function) if len(new_trigger['pdisable_trigger']) > 0: for trigger in new_trigger['pdisable_trigger']: functions.append('pdisable_trigger ' + trigger) if len(functions) == 0: print "load_triggers_from_xml: No function defined for '%s' in %s" % new_trigger['name'], file #print "load_triggers_from_xml: No function defined for generic trigger in %s" % file core.add_trigger( new_trigger['name'], new_group['name'], new_trigger['type'], new_trigger['pattern'], functions, new_trigger['enabled'], ) # Done! return new_group['name']
import os.path import requests from requests.auth import HTTPBasicAuth from StringIO import StringIO from utils import str_to_bool try: from settings import * except ImportError: GDP_BACKUP_HOST = os.environ['GDP_BACKUP_HOST'] GDP_BACKUP_USER = os.environ['GDP_BACKUP_USER'] GDP_BACKUP_PASSWORD = os.environ['GDP_BACKUP_PASSWORD'] GDP_BACKUP_FORMAT = os.environ['GDP_BACKUP_FORMAT'] GDP_BACKUP_IS_INCREMENTIVE = str_to_bool(os.environ['GDP_BACKUP_IS_INCREMENTIVE']) GDP_BACKUP_IS_NEW_FOLDER_PER_RUN = str_to_bool(os.environ['GDP_BACKUP_IS_NEW_FOLDER_PER_RUN']) GDP_BACKUP_DIR = os.environ['GDP_BACKUP_DIR'] GDP_BACKUP_STATUS_FILE = os.environ['GDP_BACKUP_STATUS_FILE'] def get_backup_folder(): files_dir = GDP_BACKUP_DIR if GDP_BACKUP_IS_NEW_FOLDER_PER_RUN: files_dir = '%s/%s' % (GDP_BACKUP_DIR, datetime.datetime.now().strftime('%m%d%y%H%M%S')) #files_dir = '%s/%s' % (GDP_BACKUP_DIR, datetime.datetime.now().strftime('%y%m%d%H%M%S')) if not os.path.exists(files_dir): os.makedirs(files_dir) logging.debug("Backup folder created: %s" % files_dir) return files_dir
def bootstrap(): global config sys.stdout.write("\nAvalon SAGE %s\n" % VERSION) sys.stdout.write("(c) 2010-2011 Matt Davis, Paul Shahid, Todd Wilson, (C) 2013 Andrew Johnson\nAll rights reserved, all wrongs revenged.\n\n") if debug_mode: sys.stdout.write(" >>> DEBUGGING MODE <<<\n\n") # Load Config config = ConfigParser.ConfigParser() if os.path.isfile(config_file) is False: sys.exit('Error: Unable to locate your configuration file (%s)' % config_file) config.read(config_file) # Pull config out of file and into dictionary "options" options = { # general 'class': config_get('general', 'class'), 'name': config_get('general', 'name'), 'unknown_affliction_ticks': config_getint('general', 'unknown_affliction_ticks'), # connection 'host': config_get('connection', 'host'), 'port': config_getint('connection', 'port'), 'listen_on': config_getint('connection', 'listen_on'), # display 'newline_on_ga': config_getboolean('display', 'newline_on_ga'), #logging 'logging_enabled': config_getboolean('logging', 'logging_enabled'), # healing 'health_sip_percentage': config_getint('healing', 'health_sip_percentage'), 'mana_sip_percentage': config_getint('healing', 'mana_sip_percentage'), 'eat_lestagii': config_getboolean('healing', 'eat_lestagii'), 'basic_lestagii_health_percentage': config_getint('healing', 'basic_lestagii_health_percentage'), 'combat_lestagii_health_percentage': config_getint('healing', 'combat_lestagii_health_percentage'), 'basic_lestagii_mana_percentage': config_getint('healing', 'basic_lestagii_mana_percentage'), 'combat_lestagii_mana_percentage': config_getint('healing', 'combat_lestagii_mana_percentage'), 'herb_balance_delay': config_getfloat('healing', 'herb_balance_delay'), # pipes 'elm': config_get_optional('pipes', 'elm'), 'artyelm': config_get_optional('pipes', 'artyelm'), 'malloran': config_get_optional('pipes', 'malloran'), 'artymalloran': config_get_optional('pipes', 'artymalloran'), 'valerian': config_get_optional('pipes', 'valerian'), 'artyvalerian': config_get_optional('pipes', 'artyvalerian'), 'min_elm': config_getint('pipes', 'min_elm'), 'min_valerian': config_getint('pipes', 'min_valerian'), 'min_malloran': config_getint('pipes', 'min_malloran'), # antiillusion 'focus_on_impatience_mana_threshold': config_getint('antiillusion', 'focus_on_impatience_mana_threshold'), } config_modules = config.items('modules') config_herbsout = config.items('herbsout') default_sections = ['connection', 'general', 'display', 'logging', 'healing', 'pipes', 'herbsout', 'antiillusion', 'modules'] classdefssections = [x for x in config.sections() if x not in default_sections] classdeflist = [config.items(section) for section in classdefssections] classdefs = [] for x in classdeflist: for y in x: classdefs.append(y) del default_sections, classdefssections, classdeflist config = options options['herbsout'] = {} for herb in config_herbsout: options['herbsout'][herb[0]] = int(herb[1]) player.name = config['name'] player.lestagii_health_percentage = config['basic_lestagii_health_percentage'] player.lestagii_mana_percentage = config['combat_lestagii_mana_percentage'] player.combat_class = config['class'] if config['artyelm'] == None: player.elm_pipe = config['elm'] player.arty_elm = False else: player.elm_pipe = config['artyelm'] player.arty_elm = True if config['artymalloran'] == None: player.malloran_pipe = config['malloran'] player.arty_malloran = False else: player.malloran_pipe = config['artymalloran'] player.arty_malloran = True if config['artyvalerian'] == None: player.valerian_pipe = config['valerian'] player.arty_valerian = False else: player.valerian_pipe = config['artyvalerian'] player.arty_valerian = True _reset_deflist() for option in classdefs: if option[0] == 'simultaneity': player.simultaneity = utils.str_to_bool(option[1]) continue elif option[0] == 'purity_necklace': continue # need to actually make that... flags = option[1] try: player.basic[option[0]] = utils.str_to_bool(flags[0]) player.keepup_basic[option[0]] = utils.str_to_bool(flags[1]) player.combat[option[0]] = utils.str_to_bool(flags[2]) player.keepup_combat[option[0]] = utils.str_to_bool(flags[3]) player.starburst[option[0]] = utils.str_to_bool(flags[4]) player.keepup_starburst[option[0]] = utils.str_to_bool(flags[5]) except: sys.exit('Invalid configuration for "' + option[0] + '".') path = sys.path[0] sys.stdout.write("> Loading configuration (%s)\n" % config_file) sys.stdout.write("> Loading Triggers\n") loadXML.load_xml_triggers(path + '/triggers/') sys.stdout.write("> Registering Trigger Dispatch Functions\n") Triggers.trigger_dispatch = loadDispatch.load_dir(path + '/inc/trigger_dispatch/') sys.stdout.write("> Loading Aliases\n") loadXML.load_xml_aliases(path + '/aliases/') sys.stdout.write("> Registering Alias Dispatch Functions\n") Aliases.alias_dispatch = loadDispatch.load_dir(path + '/inc/alias_dispatch/') """ Load modules from config and bootstrap them """ sys.stdout.write("> Loading Modules\n") for cm in config_modules: if cm[1] == 'enabled': module = cm[0] load_module(module, True) """ Check all functions called by triggers/aliases and make sure they have dispatch functions available to them. """ error = False for group in triggers: for trigger in triggers[group]: functions = triggers[group][trigger].function for function in functions: if " " in function: function = function.split(" ") if function[0] == 'pfunction': function = function[1] else: function = function[0] if function not in Triggers.trigger_dispatch: sys.stdout.write("Error: trigger '%s::%s' calls unavailable trigger dispatch function '%s'\n" % (group, trigger, function)) error = True if error: sys.exit() for group in aliases: for alias in aliases[group]: functions = aliases[group][alias].function for function in functions: if " " in function: function = function.split(" ")[0] if function not in Aliases.alias_dispatch: sys.stdout.write("Error: alias '%s::%s' calls unavailable alias dispatch function '%s'\n" % (group, alias, function)) error = True if error: sys.exit() """ Ensure we don't have duplicates in triggers and aliases. This is the quick and dirty way...could massively optmize this. """ check_dict = {} for group in trigger_groups: check_dict[group] = check_dict.get(group, 0) + 1 for item in check_dict: if check_dict[item] > 1: error = True sys.stdout.write("Error: duplicate trigger group: " + item + "\n") check_dict = {} for group in alias_groups: check_dict[group] = check_dict.get(group, 0) + 1 for item in check_dict: if check_dict[item] > 1: error = True sys.stdout.write("Error: duplicate alias group: " + item + "\n") trigger_list = [] for group in triggers: for trigger in triggers[group]: trigger_list.append(trigger) check_dict = {} for trigger in trigger_list: check_dict[trigger] = check_dict.get(trigger, 0) + 1 for item in check_dict: if check_dict[item] > 1: error = True sys.stdout.write("Error: duplicate trigger: " + item + "\n") alias_list = [] for group in aliases: for alias in aliases[group]: alias_list.append(alias) check_dict = {} for alias in alias_list: check_dict[alias] = check_dict.get(alias, 0) + 1 for item in check_dict: if check_dict[item] > 1: error = True sys.stdout.write("Error: duplicate alias: " + item + "\n") if error: sys.exit() if player.combat_class == 'monk': enable_trigger_group('monk_balances') sys.stdout.write("> Initializing Twisted\n") import telnetProxy factory = telnetProxy.Factory() factory.protocol = telnetProxy.TelnetServer reactor.listenTCP(config['listen_on'], factory) if console_enabled: reactor.callWhenRunning(interact, stopReactor=True) reactor.addSystemEventTrigger("before", "shutdown", shutdown) reactor.run()
def once_complex(self, dc, one_dc): set_title = self.start_btn.update_query update_status = self.update_status need_number = one_dc['need_number'] num_file = one_dc["number_file"] num_size = one_dc['number_size'] num_join_str = one_dc['number_join_str'] num_join_short_str = one_dc['number_join_short_str'] if not num_join_short_str: num_join_short_str = '' else: num_join_short_str = " " + num_join_short_str num_second = 0 is_iqy = True if num_join_str == '爱奇艺备案号' else False raw_mp4 = one_dc['rawMP4'] i = one_dc['index'] number_second = int(dc["number_second"]) total = one_dc['total'] out_dir = one_dc['output_dir'] temp_dir = one_dc['temp_dir'] pt_second = one_dc['pt_second'] pw_second = one_dc['pw_second'] pt_out_file = one_dc['pt_out_file'] pw_out_file = one_dc['pw_out_file'] frame_size = one_dc['frame_size'] water_size = one_dc['water_size'] rad_var = dc['fps'] if rad_var == 2: fps = '24' elif rad_var == 3: fps = '25' elif rad_var == 4: fps = '30' else: fps = '0' target_fps = fps radio_select_var = dc["bit"] pt_file = dc["pt_file"] pw_file = dc["pw_file"] frame_file = dc["frame_file"] watermark_file = dc["watermark_file"] pt_select = dc['pt_select'] pw_select = dc['pw_select'] need_frame = dc["frame_select"] need_watermark = dc["watermark_select"] double_fix_select = utils.str_to_bool(dc["select_double_fix"]) select_30m = utils.str_to_bool(dc["select_30m"]) fast_mode_select = False # fast_mode_select = dc['fast_mode_select'] # skip_content_mp4 = False count = i + 1 set_title("") format_str = "(%d/%d)" % (count, total) + ' %s' arr = utils.get_file_names(raw_mp4) f_name = arr[1] f_type = arr[2] f_full_name = f_name + f_type out_file_type = ".mpg" if select_30m else ".mp4" temp_video = temp_dir + "-" + f_name + out_file_type final_video = out_dir + f_name + out_file_type if need_number and num_join_str: temp_path = Path(out_dir) / num_join_str temp_path = str(temp_path) + os.sep utils.make_dir(temp_path) final_video = temp_path + f_name + out_file_type vb_str = "" need_same_bit_rate = False # 1) 转正片视频 set_title(format_str % f_full_name) update_status(i, '10%' + num_join_short_str) # 匹配 尺寸和fps tdc = ff.get_video_info(raw_mp4, False) v_size = tdc["v_size"] if tdc["v_size"] else "1920x1080" tdc["v_size"] = v_size fps = tdc["fps"] if tdc["fps"] else "24" tdc["fps"] = fps if target_fps == '0' else target_fps duration = tdc['duration'] if tdc["duration"] else '0' duration = float(duration) if is_iqy: vb_str = "8M" else: # 码率 部分 if radio_select_var == 1: # 保持 need_same_bit_rate = True # tdc["crf"] = 1 vb_str = '' elif radio_select_var == 2: # 自动 tdc["crf"] = 18 vb_str = '' if radio_select_var == 3: vb_str = "4M" elif radio_select_var == 4: vb_str = "6M" elif radio_select_var == 5: vb_str = "8M" elif radio_select_var == 6: vb_str = "10M" elif radio_select_var == 7: vb_str = "30M" obj = ff.create_obj() obj.input_file = raw_mp4 obj.output_file = temp_video obj.need_same_bit_rate = need_same_bit_rate obj.need_30m = select_30m # obj.set_video_info(tdc) # obj.fps = fps # obj.size = v_size obj.set_video_info(tdc, vb_str) if need_number: if number_second == -1: num_second = duration + pt_second + pw_second else: num_second = number_second if double_fix_select and duration: obj.time_start = 0 obj.time_to = duration duration_string = ff.millisecond_to_str(int(duration * 1000)) set_title(format_str % ("*[双倍时长修正]该视频时长:" + duration_string)) png_list = [] msg_str = '正在转换 正片(' if need_frame: png_list.append(["加幕布", frame_file, frame_size, 0]) if need_watermark: png_list.append([" 加水印", watermark_file, water_size, 0]) if need_number: t = num_second - pt_second png_list.append([" 加备案号", num_file, num_size, t]) if len(png_list): sizes = [] times = [] npngs = [] for p in png_list: msg_str += p[0] npngs.append(p[1]) sizes.append(p[2]) times.append(p[3]) png_list = npngs obj.set_overlay(png_list, sizes, times) msg_str += ')……' msg_str = msg_str.replace('()', '') set_title(format_str % msg_str) # 可以不转换片头的情况 # 没有选择任何合成功能时,会对正片进行一次转码操作,后面会进行处理 if not need_frame and not need_watermark and not need_number and not double_fix_select: skip_content_mp4 = True else: skip_content_mp4 = False update_status(i, '20%' + num_join_short_str) obj.execute() # 2) 有片头或片尾需要合成 if pt_select or pw_select: # 生成concat.txt, 并转换片头/片尾 subs = [] # 1 if pt_select: nobj = ff.create_obj() nobj.input_file = pt_file nobj.output_file = pt_out_file nobj.need_30m = select_30m nobj.need_same_bit_rate = need_same_bit_rate # nobj.fps = fps # nobj.size = v_size nobj.set_video_info(tdc, vb_str) # 需要添加备案号 msg_str = "正在转换 片头" if need_number and num_second: msg_str += '(加备案号)' if pt_second < num_second: nobj.set_overlay([num_file], [num_size]) else: nobj.set_overlay([num_file], [num_size], [pt_second]) msg_str += '……' set_title(format_str % msg_str) update_status(i, '40%' + num_join_short_str) nobj.execute() subs.append(pt_out_file) # 2 if skip_content_mp4: if fast_mode_select and ff.compare_video(raw_mp4, pt_out_file): subs.append(raw_mp4) # 让正片参与最后的拼接,但不能删除正片 msg_str = "没有水印等,不转换正片,直接进行合并" set_title(format_str % msg_str) else: # 和片头的视频参数不一致,进行一次转码 obj.set_video_info(tdc, vb_str) # 此操作能恢复之前的大多数参数 msg_str = "正在转换 正片" msg_str += '……' set_title(format_str % msg_str) update_status(i, '50%' + num_join_short_str) obj.execute() subs.append(temp_video) else: subs.append(temp_video) # 3 if pw_select: nobj = ff.create_obj() nobj.input_file = pw_file nobj.output_file = pw_out_file nobj.need_same_bit_rate = need_same_bit_rate nobj.need_30m = select_30m # nobj.fps = fps # nobj.size = v_size nobj.set_video_info(tdc, vb_str) # 需要添加备案号 msg_str = "正在转换 片尾" t = pt_second + duration if need_number and t < num_second: msg_str += '(加备案号)' new_t = num_second - t nobj.set_overlay([num_file], [num_size], [new_t]) msg_str += "……" set_title(format_str % msg_str) update_status(i, '60%' + num_join_short_str) nobj.execute() subs.append(pw_out_file) # 拼接视频 set_title(format_str % "拼接中……") update_status(i, '90%' + num_join_short_str) sub_txt = temp_dir + "concat_" + f_name + ".txt" ff.concat(subs, final_video, sub_txt) # 移除 concat.txt 和 mp4 utils.remove_file(sub_txt) if not skip_content_mp4: utils.remove_file(temp_video) else: # 没有任何选项 仅对正片进行一次转码操作 if skip_content_mp4: obj.execute() utils.move_file(temp_video, final_video) else: utils.move_file(temp_video, final_video) self.final_video = final_video update_status(i, 'OK')
def process(self, dc, _): set_title = self.start_btn.update_query lists = dc["input_files"] output_dir = dc["output_dir"] + os.sep output_dir = str(Path(output_dir)) + os.sep temp_dir = output_dir + 'tempDir' + os.sep utils.make_dir(temp_dir) utils.hide_file(temp_dir) pt_file = dc["pt_file"] pw_file = dc["pw_file"] frame_file = dc["frame_file"] watermark_file = dc["watermark_file"] number_file = dc["number_file"] number_file_2 = dc["number_file_2"] number_file_3 = dc["number_file_3"] pt_select = dc['pt_select'] pw_select = dc['pw_select'] need_frame = dc["frame_select"] need_watermark = dc["watermark_select"] need_number = dc['number_select'] need_number_2 = dc['number_select_2'] need_number_3 = dc['number_select_3'] # double_fix_select = dc["double_fix_select"] # fast_mode_select = False # fast_mode_select = dc['fast_mode_select'] rad_var = dc['fps'] if rad_var == 2: fps = '24' elif rad_var == 3: fps = '25' elif rad_var == 4: fps = '30' else: fps = '0' # 30m方案特殊处理 select_30m = utils.str_to_bool(dc["select_30m"]) if select_30m: pt_out_file = temp_dir + "--pt.mpg" pw_out_file = temp_dir + "--pw.mpg" else: pt_out_file = temp_dir + "--pt.mp4" pw_out_file = temp_dir + "--pw.mp4" pf = '' if pt_select: pf = "加片头" if pw_select: pf += "加片尾" if need_frame: pf += ",加幕布" if need_watermark: pf += ",加水印" if need_number: pf += ",加备案号" pf = pf.strip(', ') self.titlePrefix = pf frame_size = '0x0' water_size = '0x0' number_size = '0x0' number_size_2 = '0x0' number_size_3 = '0x0' if need_frame and frame_file: frame_size = utils.get_image_size2(frame_file) if need_watermark and watermark_file: water_size = utils.get_image_size2(watermark_file) if need_number and number_file: number_size = utils.get_image_size2(number_file) if need_number_2 and number_file_2: number_size_2 = utils.get_image_size2(number_file_2) if need_number_3 and number_file_3: number_size_3 = utils.get_image_size2(number_file_3) # 片头持续时间 pt_second = 0 if pt_select and pt_file: print(pt_file) tdc = ff.get_video_info(pt_file, False) # 匹配 尺寸和fps pt_second = tdc['duration'] if tdc["duration"] else '0' pt_second = float(pt_second) # 片尾持续时间 pw_second = 0 if pw_select and pw_file: tdc = ff.get_video_info(pw_file, False) # 匹配 尺寸和fps pw_second = tdc['duration'] if tdc["duration"] else '0' pw_second = float(pw_second) seq = (['index', -1], ['total', 0], ['rawMP4', ''], ['output_dir', ''], ['temp_dir', ''], ['pt_second', 0], ['pw_second', 0], ['pt_out_file', ''], ['pw_out_file', ''], ['frame_size', ''], ['water_size', ''], ['need_number', False], ['number_file', ''], ['number_size', ''], ['number_join_str', ''], ['number_join_short_str', ''], ['fps', 24]) # one_dc = dict.fromkeys(seq, "") one_dc = {} for key_value in seq: one_dc.setdefault(key_value[0], key_value[1]) one_dc['total'] = len(lists) one_dc['output_dir'] = output_dir one_dc['temp_dir'] = temp_dir one_dc['pt_second'] = pt_second one_dc['pw_second'] = pw_second one_dc['pt_second'] = pt_second one_dc['pt_out_file'] = pt_out_file one_dc['pw_out_file'] = pw_out_file one_dc['fps'] = fps if need_frame and frame_file: one_dc['frame_size'] = frame_size if need_watermark and watermark_file: one_dc['water_size'] = water_size for i in range(len(lists)): one_dc['index'] = i one_dc['rawMP4'] = lists[i] if need_number: one_dc['number_join_str'] = '' one_dc['number_join_short_str'] = '' one_dc['need_number'] = need_number one_dc['number_file'] = number_file one_dc['number_size'] = number_size self.once_complex(dc, one_dc) if need_number_2: one_dc['number_join_str'] = '爱奇艺备案号' one_dc['number_join_short_str'] = '爱奇艺' one_dc['need_number'] = need_number_2 one_dc['number_file'] = number_file_2 one_dc['number_size'] = number_size_2 self.once_complex(dc, one_dc) if need_number_3: one_dc['number_join_str'] = '腾讯备案号' one_dc['number_join_short_str'] = '腾讯' one_dc['need_number'] = need_number_3 one_dc['number_file'] = number_file_3 one_dc['number_size'] = number_size_3 self.once_complex(dc, one_dc) # 没有备案号任务 只执行一次 if not need_number and \ not need_number_2 and \ not need_number_3: one_dc['need_number'] = False # one_dc['number_file'] = number_file # one_dc['number_size'] = number_size # one_dc['number_join_str'] = '' # one_dc['number_join_short_str'] = '' self.once_complex(dc, one_dc) # 删除临时文件 utils.remove_file(pt_out_file) utils.remove_file(pw_out_file) print("完成!\n输出目录:" + output_dir) set_title("操作结束!") set_title("") # 自动打开目录 if self.final_video and os.path.exists(self.final_video): utils.open_file(self.final_video, True) else: utils.open_dir(output_dir) self.t1 = "" self.lock_btn(False) # 检查并执行关机 self.cb_shutdown.shutdown()
def train(config, model_dir): """Given a Configuration object containing training settings and hyperparameters, the train method launches a Trainer instance which trains a neural network model. Arguments: config {Configuration} -- Configuration object of settings, from a JSON file. model_dir {string} -- Path to the target directory of logs and results Returns: loss {double} -- The final validation or training loss, depending on the Trainer object. """ # Record time now = datetime.now() # Set seed if config.cuda: torch.cuda.manual_seed(2222) else: torch.manual_seed(2222) params = config.params pretrain = config.pretrain data_path = config.data_path source = config.source mode = config.mode trainer_name = config.trainer_name model_name = config.model_name optimizer_name = config.optimizer_name n_epochs = params.n_epochs lr = params.lr momentum = params.momentum batch_size = params.batch_size use_transform = utils.str_to_bool(config.transform) use_val = utils.str_to_bool(config.validation) # Define a loss function. reduction='none' is elementwise loss, later summed manually criterion = nn.MSELoss(reduction='none') ############### INITIALISE MODEL AND OPTIMIZER ###################### # Define a model and optimizer pair model, optimizer = utils.get_model_and_optimizer(model_name, optimizer_name, pretrain, params) ############## GET DATALOADERS ######################## # Get dataset of recovery curves logging.info("Loading the datasets...") dataset = utils.get_dataset(source, data_path, model_dir, mode, use_transform, params) logging.info("- Loading complete.") # Initialize a Regressor training object logging.info("Initializing trainer object...") trainer = utils.get_trainer(trainer_name, model, config, criterion, optimizer, dataset, model_dir) logging.info("- Initialization complete.") ################ TRAIN THE MODEL ###################### logging.info("Starting training for {} epoch(s)...".format(n_epochs)) trainer.train() logging.info("- Training complete.") torch.save(trainer.model, os.path.join(model_dir, now.strftime("%Y%m%d-%H%M") + ".pt")) return trainer.loss
print('z_noise', z_noise) rlr = float(rlr) print('rlr', rlr) dglr = float(dglr) print('dglr', dglr) sslr = float(sslr) print('sslr', sslr) drop = float(drop) print('drop', drop) cfy = float(cfy) print('cfy', cfy) cfz = float(cfz) print('cfz', cfz) dataset = dataset print('dataset', dataset) decrease = xx.str_to_bool(decrease) print('decrease', decrease) n_train = int(n_train) print('n_train', n_train) n_test = int(n_test) print('n_test', n_test) epochs = int(epochs) print('epochs', epochs) ss_labels = int(ss_labels) print('ss_labels', ss_labels) seed = int(seed) print('seed', seed) rep = int(rep) print('rep', rep) name = '{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}_{}'.\
import os import logging import json from utils import str_to_bool, ts_to_date import requests from flask import Flask, request, Response, abort APP = Flask(__name__) LOG_LEVEL = os.environ.get('LOG_LEVEL', "INFO") API_PATH = os.environ.get('API_BASE_PATH') CONVERT_TS_TO_DATE_STR = str_to_bool(os.environ.get('CONVERT_TS_TO_DATE_STR', 'true')) if not API_PATH: logging.error("API_PATH required") exit(1) # at least Ocp-Apim-Subscription-Key header with API access key must be provided HEADERS = json.loads(os.environ.get("HEADERS", '{}')) if not HEADERS.get('Ocp-Apim-Subscription-Key'): logging.error("Subscription key not found") exit(1) PORT = os.environ.get('PORT', 5001)