def checkHTTPS(self, userData): util.debug('[SC] kontrolujem nastavenia HTTPS s WS [%s] [%s]' % (getSetting('ws_usessl'), userData.find('wants_https_download').text)) toggle = False if getSettingAsBool('ws_usessl') is not True and userData.find( 'wants_https_download').text == '1': toggle = True elif getSettingAsBool('ws_usessl') is True and userData.find( 'wants_https_download').text == '0': toggle = True if toggle: headers, req = self._create_request('/', {'wst': self.token}) try: util.info('[SC] userData menim nastavenie http(s)') data = post( self._url('api/toggle_https_download/'), req, headers=headers, output="content") util.debug('[SC] zmena: %s' % str(data)) except: self.clearToken() return False pass
def get_info(url): cached = get_cached_info(url) if cached: return cached else: info = _empty_info() util.info('Not in cache : '+url) try: page = util.request(url,headers={'Referer':BASE_URL,'User-Agent':util.UA}) except: util.error('Unable to read page '+url) traceback.print_exc() info['url'] = url return info info['title'] = _get_title(page) info['search-title'] = _search_title(page) info['url'] = url info['trailers_url'] = url.rstrip('/')+'/videa/' info['cast'] = _cast(page) info['genre'] = _genre(page) info['img'] = _get_img(page) info['plot'] = _plot(page) country,info['year'] = _origin(page) info['percent'],info['rating'] = _rating(page) info['director'] = _director(page) info['votes'] = _votes(page) _validate(info) set_info(info) return info
def run(self, circuit): """Given a logisim.Circuit object, set its input pins, evaluate the circuit, and determine if its output matches the expected output. """ from logisim.errors import NoInputsError util.info("running eval test on '{}'".format(circuit.name)) try: output_vals = circuit.eval(self.input) except NoInputsError as e: desc = "a component is missing an input value" return {'deduction': self.deduction, 'description': desc, 'notes': [str(e)]} failed = False for label, value in self.output.items(): pin = _get_pin_with_label(output_vals.keys(), label) if output_vals[pin] != value: failed = True break if failed: desc = _build_description(self.input, self.output).split('\n') desc += ["your circuit produced:"] desc += _build_description({}, output_vals).split('\n') return {'deduction': self.deduction, 'description': "did not produce the correct output", 'notes': desc}
def do_POST(self): """Handle POST requests to the API endpoint""" global endpoint parsed_path = urlparse(self.path) if "/api/v1/post_parameters" in parsed_path: self.send_response(200) # TODO: Security? self.send_header("Access-Control-Allow-Origin", "*") self.end_headers() length = int(self.headers["Content-Length"]) post_data = self.rfile.read(length).decode("utf-8") #post_data = self.rfile.read(length) # Parse data from POST print('Got a post ') print(type(post_data)) print(post_data) new_data = parse_qs(post_data) print(type(new_data)) for x in new_data: print(x) print(new_data[x]) params.new_num_active_workers = int(new_data['num_workers'][0]) #change_num_active_workers() else: util.info("POST sent to " + str(parsed_path[2]))
def run_hooks_for(trigger): from sys import exit from os.path import sep from subprocess import call global _triggers if trigger not in _triggers: raise ValueError("unknown trigger: '" + str(trigger) + "'") hooks = list(set(_hooks[trigger]) - set(_hooks_done[trigger])) num_done = 0 if len(hooks) > 0: util.info("running hooks for trigger '" + str(trigger) + "'") for fname in hooks: rv = call(config.hooks_dir + sep + fname, env=_create_env()) _hooks_done[trigger].append(fname) num_done += 1 if rv != 0: util.error("hook '" + str(fname) + "' exited abnormally") util.exit(util.ERR_ABNORMAL_HOOK_EXIT) util.info("successfully ran " + str(num_done) + " " + \ util.plural('hook', num_done))
def policy(self, state, legal_moves): """The policy of picking an action based on their weights.""" if not legal_moves: return None if not self.minimax_enabled: # don't use minimax if we're in learning mode best_move, _ = best_move_val( self.model.predict(numpify(state)), legal_moves ) return best_move else: next_states = {self.reversi.next_state( state, move): move for move in legal_moves} move_scores = [] for s in next_states.keys(): score = self.minimax(s) move_scores.append((score, s)) info('{}: {}'.format(next_states[s], score)) best_val = -float('inf') best_move = None for each in move_scores: if each[0] > best_val: best_val = each[0] best_move = next_states[each[1]] assert best_move is not None return best_move
def _get_file_url_anonymous(self,page,post_url,headers,captcha_cb): capdata = json.loads(util.request(self._url('reloadXapca.php'))) captcha = capdata['image'] if not captcha.startswith('http'): captcha = 'http:' + captcha sound = capdata['sound'] if not sound.startswith('http'): sound = 'http:' + sound # ask callback to provide captcha code self.info('Asking for captcha img %s' % captcha) code = captcha_cb({'id':captcha,'img': captcha,'snd':sound}) if not code: self.info('Captcha not provided, done') return ts = re.search('<input type=\"hidden\" name=\"ts\".+?value=\"([^\"]+)"',page,re.IGNORECASE | re.DOTALL) cid = re.search('<input type=\"hidden\" name=\"cid\".+?value=\"([^\"]+)"',page,re.IGNORECASE | re.DOTALL) sign = re.search('<input type=\"hidden\" name=\"sign\".+?value=\"([^\"]+)"',page,re.IGNORECASE | re.DOTALL) has = capdata['hash'] salt = capdata['salt'] timestamp = capdata['timestamp'] token = re.search('<input type=\"hidden\" name=\"_token_\".+?value=\"([^\"]+)"',page,re.IGNORECASE | re.DOTALL) if not (sign and ts and cid and has and token): util.error('[uloz.to] - unable to parse required params from page, plugin needs fix') return request = {'captcha_type':'xcapca','hash':has,'salt':salt,'timestamp':timestamp,'ts':ts.group(1),'cid':cid.group(1),'sign':sign.group(1),'captcha_value':code,'do':'downloadDialog-freeDownloadForm-submit','_token_':token.group(1)} req = urllib2.Request(post_url,urllib.urlencode(request)) req.add_header('User-Agent',util.UA) req.add_header('Referer',post_url) req.add_header('Accept','application/json') req.add_header('X-Requested-With','XMLHttpRequest') sessid=[] for cookie in re.finditer('(ULOSESSID=[^\;]+)',headers.get('Set-Cookie'),re.IGNORECASE | re.DOTALL): sessid.append(cookie.group(1)) req.add_header('Cookie','nomobile=1; uloztoid='+cid.group(1)+'uloztoid2='+cid.group(1)+'; '+sessid[-1]) util.info(request) try: resp = urllib2.urlopen(req) page = resp.read() headers = resp.headers except urllib2.HTTPError: # this is not OK, something went wrong traceback.print_exc() util.error('[uloz.to] cannot resolve stream url, server did not redirected us') util.info('[uloz.to] POST url:'+post_url) return try: result = json.loads(page) except: raise ResolveException('Unexpected error, addon needs fix') if not 'status' in result.keys(): raise ResolveException('Unexpected error, addon needs fix') if result['status'] == 'ok': return self._fix_stream_url(result['url']) elif result['status'] == 'error': # the only known state is wrong captcha for now util.error('Captcha validation failed, please try playing/downloading again') util.error(result) raise ResolveException('Captcha failed, try again')
def resolve(url): """ resolves given url by asking all resolvers returns None if no resolver advised to be able to resolve this url returns False if resolver did his job, but did not return any value (thus failed) returns Array of resolved objects in positive usecase """ url = util.decode_html(url) util.info('Resolving ' + url) resolver = _get_resolver(url) value = None if resolver is None: return None util.info('Using resolver \'%s\'' % str(resolver.__name__)); try: value = resolver.resolve(url) except: traceback.print_exc() if value is None: return False default = item() def fix_stream(i, url, resolver, default): """ fix missing but required values """ if 'name' not in i.keys(): i['name'] = resolver.__name__ if 'surl' not in i.keys(): i['surl'] = url for key in default.keys(): if key not in i.keys(): i[key] = default[key] [fix_stream(i, url, resolver, default) for i in value] return sorted(value, key=lambda i: i['quality'])
def download(self, urls=None): total_downloaded = 0 if urls is None: connections = [self.connect(self.host) for i in range(self.runs)] else: connections = [self.connect(h['host']) for h in urls] total_start_time = time() for current_file in self.DOWNLOAD_FILES: threads = [] for run in range(self.runs): thread = Thread( target=self.downloadthread, args=(connections[run], '%s?x=%d' % (current_file, int(time() * 1000)) if urls is None else urls[run]['url'])) thread.run_number = run + 1 thread.start() threads.append(thread) for thread in threads: try: thread.join() total_downloaded += thread.downloaded util.debug('[SC] Run %d for %s finished' % (thread.run_number, current_file)) except: pass total_ms = (time() - total_start_time) * 1000 for connection in connections: connection.close() util.info('[SC] Took %d ms to download %d bytes' % (total_ms, total_downloaded)) return total_downloaded * 8000 / total_ms
def validate_model(best): if gen_n_text_samples: print '\nGenerating %d text samples...' % gen_n_text_samples n_seed = 100 start = max(0, np.random.randint(0, training_dataset.n_words - n_seed)) seed = training_dataset.get_words()[start: start + n_seed] gen_text(seed=seed, how_many=gen_n_text_samples) print '\nValidating model...' validation_t.start() v_model.set_weights(t_model.get_weights()) v_model.reset_states() n_v_samples, gen_v = validation_data[0]() loss, _ = v_model.evaluate_generator(gen_v, n_v_samples) pp = np.exp(loss) val_elapsed, val_tot = validation_t.lap() validation_info = '''Validation result: - Model loss: %f - Perplexity: %f %s - OOV rate: %f - Validation took: %s - Total validation: %s ''' % (loss, pp, delta_str(pp, best), validation_data[1], val_elapsed, val_tot) info(validation_info) return pp
def _add_source(self, filename, sourcename, transform=None, only_a_few_lines=True): """ Update the features dict with data from the file named filename. Use the same name for the type of data source. """ self._progress("Adding source = %s" % sourcename) file = open(filename, "r") line_no = 0 for line in file: line_no += 1 if line_no % 500 == 0: self._progress("cur line = %d" % line_no) if only_a_few_lines and not self.production_run and line_no > 200: util.info("\tWARNING: stopping at line 200 of input file. Turn off for production runs.") break cur_tag = self._read_tag(line) if cur_tag in self.only_these_tags: cur_dict = self._line_to_dict(line.rstrip().split("\t"), transform=transform) if cur_dict: # that is, if cur_dict is not empty try: source_dict = self.features[cur_tag] except KeyError: source_dict = dict() try: old_dict = source_dict[sourcename] # If we get here, we need to merge the new # cur_dict with the old one. source_dict[sourcename] = self._merge_song_dicts(old_dict, cur_dict) except KeyError: # We're adding a new source. source_dict[sourcename] = cur_dict self.features[cur_tag] = source_dict file.close()
def test(cmd): global expected global got global count util.info("") util.info("- Starting " + cmd) util.info("") util.run(cmd) start = time.time() clientlist = [] expected = [] for i in range(1, NUM_CLIENT): expected.append(commands.getoutput("cat ./testdata/file%s.txt" % str(i))) commands.getoutput("rm -rf %s" % tmpfile) for i in range(0, NUM_CLIENT): client = testit("Client-" + str(i), i) clientlist.append(client) client.start() time.sleep(0.3) for client in clientlist: client.join() end = time.time() util.info("Elapsed time (in seconds): " + str(end-start)) time.sleep(CGI_SPIN_TIME + 2) res = commands.getoutput("cat %s" % tmpfile) if util.is_server_alive(cmd) == -1: util.error("Ouch! Server is dead!" " Your bounded buffered may not be well protected"); pos0 = res.find(expected[0]) pos1 = res.find(expected[1]) pos2 = res.find(expected[2]) passed = pos0 > 0 and pos1 > 0 and pos2 > 0 and pos0 < pos1 and pos1 < pos2 util.info(res) if passed: print "" print "#####################################" print "GOOD! you implement SFF correctly" print "#####################################" print "" count = count + 1 else: print "" print "#####################################" print "Oh oh! ERROR ERROR!" print "SFF is not implemented correctly" print "#####################################" print "" sys.exit(-1)
def _performance_stats(self, remove_tags_dont_work=True): self._progress("Computing performance stats.") for (tag, true_y) in self.y.items(): if tag not in self.stats: pdb.set_trace() yhat = self.yhat[tag].transpose() # make n_songs x 1 self.stats[tag]["Num Songs"] = len(yhat) # SSE self.stats[tag][r'$\text{SSE} / n$'] = numpy.sum(numpy.power((true_y-yhat),2)) / len(true_y) # precision, recall, etc. sorted_yhat = sorted([(yhat[i,0], i) for i in range(len(yhat))], reverse=True) graded = [self._in_ground_truth(true_y[i,0]) for (yhat_val, i) in sorted_yhat] try: self.stats[tag]["Baseline"] = self._random_precision(graded) self.stats[tag]["AUC"] = self._areaUnderCurve(graded) self.stats[tag]["MAP"] = self._avgPrecision(graded) self.stats[tag]["R-Prec"] = self._rPrecision(graded) self.stats[tag]["10-Prec"] = self._tenPrecision(graded) baseline = self.stats[tag]["Baseline"] if baseline > 0: self.stats[tag]["MAP/Baseline"] = self.stats[tag]["MAP"] / baseline self.stats[tag]["R-Prec/Baseline"] = self.stats[tag]["R-Prec"] / baseline self.stats[tag]["10-Prec/Baseline"] = self.stats[tag]["10-Prec"] / baseline except ValueError: util.info("WARNING: TP==0 or FP==0 for tag = %s." % tag) if remove_tags_dont_work: self._remove_tag(tag) continue # Record best and worst songs. song_list = list(self.song_lists[tag]) self.best_worst_songs[tag] = dict() index_best_song = sorted_yhat[0][1] self.best_worst_songs[tag]["Best Song"] = (self.songid_to_song[song_list[index_best_song]], 1 if true_y[index_best_song,0] else 0) index_worst_song = sorted_yhat[-1][1] self.best_worst_songs[tag]["Worst Song"] = (self.songid_to_song[song_list[index_worst_song]], 1 if true_y[index_worst_song,0] else 0)
def __init__(self, provider, settings, addon): ''' XBMContentProvider constructor Args: name (str): name of provider ''' self.provider = provider # inject current user language try: # not fully supported on Frodo provider.lang = xbmc.getLanguage(xbmc.ISO_639_1) except: provider.lang = None pass self.settings = settings # lang setting is optional for plugins if not 'lang' in self.settings: self.settings['lang'] = '0' util.info('Initializing provider %s with settings %s' % (provider.name, settings)) self.addon = addon self.addon_id = addon.getAddonInfo('id') if '!download' not in self.provider.capabilities(): self.check_setting_keys(['downloads']) self.cache = provider.cache provider.on_init()
def service(self): util.info("SOSAC Service Started") try: sleep_time = int(self.getSetting("start_sleep_time")) * 1000 * 60 except: sleep_time = self.sleep_time pass self.sleep(sleep_time) try: self.last_run = float(self.cache.get("subscription.last_run")) except: self.last_run = time.time() self.cache.set("subscription.last_run", str(self.last_run)) pass if not xbmc.abortRequested and time.time() > self.last_run: self.evalSchedules() while not xbmc.abortRequested: # evaluate subsciptions every 10 minutes if(time.time() > self.last_run + 600): self.evalSchedules() self.last_run = time.time() self.cache.set("subscription.last_run", str(self.last_run)) self.sleep(self.sleep_time) util.info("SOSAC Shutdown")
def findstreams(data,regexes): resolvables = {} resolved = [] # keep list of found urls to aviod having duplicates urls = [] error = False for regex in regexes: for match in re.finditer(regex,data,re.IGNORECASE | re.DOTALL): url = filter_resolvable(match.group('url')) if url: util.info('Found resolvable %s ' % url) resolvables[url] = None for rurl in resolvables: streams = resolve(rurl) if streams == []: util.debug('There was an error resolving '+rurl) error = True if not streams == None: if len(streams) > 0: for stream in streams: resolved.append(stream) if error and len(resolved) == 0: return None if len(resolved) == 0: return {} resolved = sorted(resolved,key=lambda i:i['quality']) resolved = sorted(resolved,key=lambda i:len(i['quality'])) resolved.reverse() return resolved
def resolve(self, ident, download_type=None): params = {'ident': ident, 'wst': self.token} if None is not download_type: params.update({ 'download_type': download_type, 'device_uuid': getSetting('uid'), 'device_res_x': infoLabel('System.ScreenWidth'), 'device_res_y': infoLabel('System.ScreenHeight'), }) headers, req = self._create_request('/', params) util.info(headers) util.info(req) try: data = post( self._url('api/file_link/'), req, headers=headers, output="content") xml = ET.fromstring(data) if not xml.find('status').text == 'OK': self.clearToken() util.error( '[SC] Server returned error status, response: %s' % data) raise ResolveException(xml.find('message').text) return xml.find('link').text except Exception as e: self.clearToken() raise ResolveException(e)
def resolve(url): url = util.decode_html(url) util.info('Resolving '+url) resolver = _get_resolver(url) value = None if resolver == None: return None util.debug('Using resolver '+str(resolver.__name__)); try: value = resolver.resolve(url) except: traceback.print_exc() if value == None: return [] default = item() # fix missing but required values def fix_stream(i,url,resolver,default): if not 'name' in i.keys(): i['name'] = resolver.__name__ if not 'surl' in i.keys(): i['surl'] = url for key in default.keys(): if not key in i.keys(): i[key] = default[key] [fix_stream(i,url,resolver,default) for i in value] return sorted(value,key=lambda i:i['quality'])
def test(cmd): util.info("") util.info("- Starting " + cmd) util.info("") util.run(cmd) commands.getoutput("rm -rf " + file1) util.info("") util.info("- Sending ./testclient localhost 2010 /testdata/file1.txt") res = commands.getoutput("./testclient localhost 2010 /testdata/file1.txt") arrival = util.get_stat2(res, "Stat-req-arrival") dispatch = util.get_stat2(res, "Stat-req-dispatch") read = util.get_stat2(res, "Stat-req-read") complete = util.get_stat2(res, "Stat-req-complete") print "" print "dispatch = " + str(dispatch) print "read = " + str(read) print "complete = " + str(complete) if dispatch >= 0 and read >=0 and complete >= 0 and dispatch + read <= complete: util.good("You passed this test") else: util.error("Expected dispatch >= 0 and read >=0 and complete >= 0 and" " dispatch + read <= complete:")
def service(self): util.info("Start") try: sleep_time = int(self.getSetting("start_sleep_time")) * 1000 * 60 except: sleep_time = self.sleep_time pass self.sleep(sleep_time) try: self.last_run = float(self.cache.get("subscription.last_run")) # time.time() except: self.last_run = time.time() self.cache.set("subscription.last_run", str(self.last_run)) pass if time.time() > self.last_run + 24 * 3600: self.evalSchedules() while not xbmc.abortRequested: if(time.time() > self.last_run + 24 * 3600): self.evalSchedules() self.last_run = time.time() self.cache.set("subscription.last_run", str(self.last_run)) self.sleep(self.sleep_time) util.info("Koncim")
def download(addon, filename, url, local, notifyFinishDialog=True, headers={}): try: util.info('Downloading %s to %s' % (url, local)) except: util.info('Downloading ' + url) local = xbmc.makeLegalFilename(local) try: filename = util.replace_diacritic(util.decode_html(filename)) except: filename = 'Video soubor' icon = os.path.join(addon.getAddonInfo('path'), 'icon.png') notifyEnabled = addon.getSetting('download-notify') == 'true' notifyEvery = addon.getSetting('download-notify-every') notifyPercent = 1 if int(notifyEvery) == 0: notifyPercent = 10 if int(notifyEvery) == 1: notifyPercent = 5 def encode(string): return u' '.join(string).encode('utf-8') def notify(title, message, time=3000): try: xbmcgui.Dialog().notification(encode(title), encode(message), time=time, icon=icon, sound=False) except: traceback.print_exc() error('unable to show notification') def callback(percent, speed, est, filename): if percent == 0 and speed == 0: notify(xbmc.getLocalizedString(13413), filename) return if notifyEnabled: if percent > 0 and percent % notifyPercent == 0: esTime = '%ss' % est if est > 60: esTime = '%sm' % int(est / 60) message = xbmc.getLocalizedString( 24042) % percent + ' - %s KB/s %s' % (speed, esTime) notify(message, filename) downloader = Downloader(callback) result = downloader.download(url, local, filename, headers) try: if result == True: if xbmc.Player().isPlaying(): notify(xbmc.getLocalizedString(20177), filename) else: if notifyFinishDialog: xbmcgui.Dialog().ok(xbmc.getLocalizedString(20177), filename) else: notify(xbmc.getLocalizedString(20177), filename) else: notify(xbmc.getLocalizedString(257), filename) xbmcgui.Dialog().ok(filename, xbmc.getLocalizedString(257) + ' : ' + result) except: traceback.print_exc()
def run(self): clientcmd = "./testclient localhost 2010 /testdata/testfile.txt" util.info(self.clientname + ": " + clientcmd) response = commands.getoutput(clientcmd) if response.find("hey this is a test file") == -1: util.error(self.clientname + ":ouchs! client can not get the right file") print self.clientname + ":Client got expected response"
def pack(self): archive = tarfile.open(self.dest, 'w:bz2') archive.add(self.builddir, arcname = '/', recursive = True ) archive.close() util.info('%s is ready.' % self.dest)
def _info_about_r_error(self, tag_list): """ Print info when rhier... functions raise a sorting-related error. Delete all tags we're working with. """ util.info("\tERROR: Problem with R's sorting thingymajig. %s" % str(sys.exc_info())) for tag in tag_list: self._remove_tag(tag)
def get_or_make_group(conn, name): groups = conn.get_all_security_groups() group = [g for g in groups if g.name == name] if len(group) > 0: return group[0] else: info("Creating security group {name} in {region}".format(name=name, region=conn.region)) return conn.create_security_group(name, "Auto created by Tachyon deploy")
def _remove_uncommon_tags(self): feature_tags = set(self.features.keys()) ground_truth_tags = set(self.ground_truth.keys()) remove_these = feature_tags.symmetric_difference(ground_truth_tags) for tag in remove_these: self._remove_tag(tag, verbosity=0) if self.verbosity > 1: util.info("\tWARNING: Removing these tags not in intersection of features and ground truth: %s." % str(remove_these))
def ask_for_account_type(self): if len(self.provider.username) == 0: util.info('Username is not set, NOT using VIP account') return False if self.settings['vip'] == '0': util.info('Asking user whether to use VIP account') ret = xbmcgui.Dialog().yesno(self.provider.name, xbmcutil.__lang__(30010)) return ret == 1 return self.settings['vip'] == '1'
def _validate(info): dummy = _empty_info() for key in info.keys(): if not type(info[key]) == type(dummy[key]): util.info('key '+key+' is type of '+str(type(info[key]))+' does not match required type '+str(type(dummy[key]))) info[key] = dummy[key] for key in dummy.keys(): if not key in info: info[key] = dummy[key]
def set_security_group(conn, name): info("Setting up security group {} in {}".format(name, conn.region)) sg = get_or_make_group(conn, name) if sg.rules != []: warn('security group {} in {} already has rules, no modification will happen then'.format(name, conn.region)) return proto = ['tcp', 'udp'] authorized_ip = '0.0.0.0/0' # all IP for p in proto: sg.authorize(p, 0, 65535, authorized_ip)
def to_downloads(self,url): if not self.login(): util.error('[hellspy] login failed, unable to add to downloads') util.info('adding to downloads') try: util.request(self._url(url+'&do=downloadControl-favourite')) except urllib2.HTTPError: traceback.print_exc() util.error('[hellspy] failed to add to downloads') return util.info('added, DONE')
import guillotine import datafiles import config from data_sources import file_source IDLE_TIME = datetime.timedelta(seconds=1) MAX_NOFILL_TIME = datetime.timedelta(minutes=50) WARN_NOFILL_SYMS = 10 NUM_WAIT_TIMES_IN_IDLE = 3 # number of times we should wait in the idle loop for guillotine # to be initialized (get symbols etc). The time for which we wait # is this number multiplied by the 'timeout' (specified below) MKT_CLOSE_TIME = util.exchangeOpenClose()[1] util.check_include() cfg_file = os.environ['CONFIG_DIR'] + '/exec.conf' util.info("loading config file: %s" % cfg_file) trade_cfg = config.load_trade_config(cfg_file) tic2sec, sec2tic = datafiles.load_tickers(os.environ['RUN_DIR'] + "/tickers.txt") util.set_log_file() class trade_listener(object): def __init__(self, gtc): self.gtc = gtc # self.last_submitted = None self.last_submitted_time = 0 self.fs = file_source.FileSource(os.environ['RUN_DIR'] + '/orders/') self.last_idle_time = datetime.datetime.utcnow() self.last_trade_time = datetime.datetime.utcnow()
action='store_true', help='Dictates whether to load optim dict, scheduler dict, epoch_i') args = parser.parse_args() assert args.save_path is not None # Make dirs if args.load_path is not None: os.makedirs(os.path.dirname(args.load_path), exist_ok=True) os.makedirs(os.path.dirname(args.save_path), exist_ok=True) if args.log_path is not None: os.makedirs(os.path.dirname(args.log_path), exist_ok=True) DO_LOAD = args.load_path is not None and os.path.exists(args.load_path) # Start logging util.init_logger(args.log_path) info(args) # Make dataloaders trainset, testset = get_datasets(args.train, args.test) trainloader = DataLoader(trainset, batch_size=args.train_bs, shuffle=True) testloader = DataLoader(testset, batch_size=args.test_bs) # Build model master_net = network.Net(arch=[int(x) for x in args.arch.split()]) if DO_LOAD: dump = torch.load(args.load_path) epoch_i = dump['epoch_i'] if args.do_continue else 0 master_net.load_state_dict(dump['state_dict']) info('Loaded from %s' % args.load_path) else: epoch_i = 0
self.fps_counter = Label(master=self.root, bg='#000000', fg='#FFFFFF') self.fps_counter.place(anchor=SE, x=1280, y=720) self.time_ = time.time() self.frames = 0 def update(self): self.root.update() self.root.update_idletasks() self.window_manager.update() self.frames += 1 time_b = time.time() diff = time_b - self.time_ if diff > 1: self.fps_counter.config(text=str(self.frames) + " fps") self.frames = 0 self.time_ = time.time() def show(self, content): self.window_manager.show(content) if __name__ == '__main__': info("Starting Application.") window = Application() while True: window.update()
def get_data_cached(self, url, post=False): try: url.index('/json/') self._oldapi() except Exception: pass headers = { 'X-UID': self.uid, 'X-LANG': self.tr['language'], 'X-VER': sctop.API_VERSION, 'Accept': 'application/vnd.bbaron.kodi-plugin-v%s+json' % (sctop.API_VERSION), } url = self._url(url) code = None try: if post != False: util.debug("POST URL: %s %s" % (url, str(post))) (ret, code) = sctop.post(url, post, headers, "extend") self.handleHttpError(code) return ret util.info("GET x URL: %s" % url) ret = False if sctop.getSettingAsBool('usecache') is not False: util.debug("[SC] skusam cache") ret = self.cache.get(str(url)) if not ret: util.debug("[SC] url BEZ cache %s" % str(url)) (ret, code, info) = sctop.request(url, headers, "info") util.debug("[SC] code: %s %s" % (str(code), str(info))) self.handleHttpError(code, data=ret, i=info) if code == 200: ttl = datetime.timedelta(hours=2) try: util.debug("[SC] info: %s " % str(info)) if 'x-ttl' in info: ttl = datetime.timedelta( seconds=int(info.get('x-ttl'))) util.debug("[SC] mame TTL: %s" % str(ttl)) except: pass try: self.cache.cache.set(str(url), ret, expiration=ttl) except: self.cache.set(str(url), ret) else: util.debug("[SC] url z cache %s" % str(url)) util.debug("[SC] return data") return ret except Exception as e: inet = sctop.getCondVisibility('System.InternetState') util.debug("[SC] inet scinema status: %s | %s" % (str(inet), str(e))) if inet is False or inet == 0: HANDLE = int(sys.argv[1]) xbmcplugin.endOfDirectory(HANDLE, succeeded=False) sctop.dialog.ok("internet", 'Skontrolujte pripojenie na internet') return False util.error('[SC] ERROR URL: --------- %s --------' % str(traceback.format_exc())) if code is None: sctop.dialog.ok("error", url) return False
def deploy_monitoring_landscape( cfg_set: ConfigurationSet, cfg_factory: ConfigFactory, ): kubernetes_cfg = cfg_set.kubernetes() concourse_cfg = cfg_set.concourse() # Set the global context to the cluster specified in KubernetesConfig kube_ctx.set_kubecfg(kubernetes_cfg.kubeconfig()) ensure_cluster_version(kubernetes_cfg) monitoring_config_name = concourse_cfg.monitoring_config() monitoring_cfg = cfg_factory.monitoring(monitoring_config_name) monitoring_namespace = monitoring_cfg.namespace() tls_config_name = concourse_cfg.tls_config() tls_config = cfg_factory.tls_config(tls_config_name) # deploy kube-state-metrics kube_state_metrics_helm_values = create_kube_state_metrics_helm_values( monitoring_cfg=monitoring_cfg) execute_helm_deployment( kubernetes_cfg, monitoring_namespace, 'stable/kube-state-metrics', 'kube-state-metrics', kube_state_metrics_helm_values, ) # deploy postgresql exporter postgresql_helm_values = create_postgresql_helm_values( concourse_cfg=concourse_cfg, cfg_factory=cfg_factory, ) execute_helm_deployment( kubernetes_cfg, monitoring_namespace, 'stable/prometheus-postgres-exporter', 'prometheus-postgres-exporter', postgresql_helm_values, ) # deploy ingresses for kube-state-metrics, postgresql exporter monitoring_tls_secret_name = monitoring_cfg.tls_secret_name() info( 'Creating tls-secret in monitoring namespace for kube-state-metrics and postgresql...' ) create_tls_secret(tls_config=tls_config, tls_secret_name=monitoring_tls_secret_name, namespace=monitoring_namespace, basic_auth_cred=BasicAuthCred( user=monitoring_cfg.basic_auth_user(), password=monitoring_cfg.basic_auth_pwd())) ingress_helper = kube_ctx.ingress_helper() info('Create ingress for kube-state-metrics') ingress = generate_monitoring_ingress_object( secret_name=monitoring_tls_secret_name, namespace=monitoring_namespace, hosts=[monitoring_cfg.ingress_host(), monitoring_cfg.external_url()], service_name=monitoring_cfg.kube_state_metrics().service_name(), service_port=monitoring_cfg.kube_state_metrics().service_port(), ) ingress_helper.replace_or_create_ingress(monitoring_namespace, ingress) info('Create ingress for postgres-exporter') ingress = generate_monitoring_ingress_object( secret_name=monitoring_tls_secret_name, namespace=monitoring_namespace, hosts=[monitoring_cfg.ingress_host(), monitoring_cfg.external_url()], service_name=monitoring_cfg.postgresql_exporter().service_name(), service_port=monitoring_cfg.postgresql_exporter().service_port(), ) ingress_helper.replace_or_create_ingress(monitoring_namespace, ingress)
inpath="/".join((os.environ["RUN_DIR"],"calcres")) outpath="/".join((os.environ["ROOT_DIR"],"reports",args.strat,"dailylive",args.date)) elif args.mode == "old": inpath="/".join((os.environ["RUN_DIR"],"oldsystem","calcres")) outpath="/".join((os.environ["ROOT_DIR"],"reports",args.strat,"dailylive",args.date)) else: print "Unknown mode!" exit(1) #create the outpath if it doesn't exist try: os.makedirs(outpath) except: pass util.info("Looking for calcres files in {}".format(inpath)) infiles = os.popen("ls -rt {}/calcres.*.txt.gz | tail -1".format(inpath)).readlines() for infile in [x.strip() for x in infiles]: outfile = infile[infile.rindex("/"):].split(".") if args.mode != "old": outfile.insert(1,"stats") else: outfile.insert(1,"old.stats") outfile=".".join(outfile[:-1]) outfile = outpath+"/"+outfile calcresStats(infile, outfile,True if args.mode=="old" else False) util.cat(outfile) if args.forecastStats: if args.infile is not None and args.outfile is not None: forecastStats(args.infile,args.outfile,args.configfile) exit(0)
def read_data(self): """Provides images and camera intrinsics.""" with tf.name_scope('data_loading'): with tf.name_scope('enqueue_paths'): seed = random.randint(0, 2**31 - 1) self.file_lists = self.compile_file_list( self.data_dir, self.input_file) image_paths_queue = tf.train.string_input_producer( self.file_lists['image_file_list'], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None)) seg_paths_queue = tf.train.string_input_producer( self.file_lists['segment_file_list'], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None)) cam_paths_queue = tf.train.string_input_producer( self.file_lists['cam_file_list'], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None)) img_reader = tf.WholeFileReader() _, image_contents = img_reader.read(image_paths_queue) seg_reader = tf.WholeFileReader() _, seg_contents = seg_reader.read(seg_paths_queue) if self.file_extension == 'jpg': image_seq = tf.image.decode_jpeg(image_contents) seg_seq = tf.image.decode_jpeg(seg_contents, channels=3) elif self.file_extension == 'png': image_seq = tf.image.decode_png(image_contents, channels=3) seg_seq = tf.image.decode_png(seg_contents, channels=3) with tf.name_scope('load_intrinsics'): cam_reader = tf.TextLineReader() _, raw_cam_contents = cam_reader.read(cam_paths_queue) rec_def = [] for _ in range(9): rec_def.append([1.0]) raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def) raw_cam_vec = tf.stack(raw_cam_vec) intrinsics = tf.reshape(raw_cam_vec, [3, 3]) with tf.name_scope('convert_image'): image_seq = self.preprocess_image( image_seq) # Converts to float32. if self.random_color: with tf.name_scope('image_augmentation'): image_seq = self.augment_image_colorspace(image_seq) image_stack = self.unpack_images(image_seq) seg_stack = self.unpack_images(seg_seq) if self.flipping_mode != FLIP_NONE: random_flipping = (self.flipping_mode == FLIP_RANDOM) with tf.name_scope('image_augmentation_flip'): image_stack, seg_stack, intrinsics = self.augment_images_flip( image_stack, seg_stack, intrinsics, randomized=random_flipping) if self.random_scale_crop: with tf.name_scope('image_augmentation_scale_crop'): image_stack, seg_stack, intrinsics = self.augment_images_scale_crop( image_stack, seg_stack, intrinsics, self.img_height, self.img_width) with tf.name_scope('multi_scale_intrinsics'): intrinsic_mat = self.get_multi_scale_intrinsics( intrinsics, self.num_scales) intrinsic_mat.set_shape([self.num_scales, 3, 3]) intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat) intrinsic_mat_inv.set_shape([self.num_scales, 3, 3]) if self.imagenet_norm: im_mean = tf.tile(tf.constant(IMAGENET_MEAN), multiples=[self.seq_length]) im_sd = tf.tile(tf.constant(IMAGENET_SD), multiples=[self.seq_length]) image_stack_norm = (image_stack - im_mean) / im_sd else: image_stack_norm = image_stack with tf.name_scope('batching'): if self.shuffle: (image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv) = tf.train.shuffle_batch( [ image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv ], batch_size=self.batch_size, capacity=QUEUE_SIZE + QUEUE_BUFFER * self.batch_size, min_after_dequeue=QUEUE_SIZE) else: (image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv) = tf.train.batch( [ image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv ], batch_size=self.batch_size, num_threads=1, capacity=QUEUE_SIZE + QUEUE_BUFFER * self.batch_size) logging.info('image_stack: %s', util.info(image_stack)) return (image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv)
def upload_image( self, container_image: ContainerImage, component: Component, ) -> UploadResult: metadata = self._metadata( container_image=container_image, component=component, omit_version=False, ) upload_result = partial(UploadResult, container_image=container_image, component=component) # check if the image has already been uploaded for this component scan_result = self.retrieve_scan_result( container_image=container_image, component=component, ) reference_results = [ self.retrieve_scan_result( container_image=container_image, component=component, group_id=group_id, ) for group_id in self._reference_group_ids ] reference_results = [r for r in reference_results if r] # remove None entries if scan_result: reference_results.insert(0, scan_result) # collect old triages in order to "transport" them after new upload (may be None) triages = self._existing_triages(analysis_results=reference_results, ) upload_action = self._determine_upload_action( container_image=container_image, scan_result=scan_result) if not upload_action.upload and not upload_action.rescan: # early exit (nothing to do) return upload_result( status=UploadStatus.SKIPPED, result=scan_result, ) if upload_action.upload: info( f'uploading to protecode: {container_image.image_reference()}') image_data_fh = retrieve_container_image( container_image.image_reference(), outfileobj=tempfile.NamedTemporaryFile(), ) if self._upload_registry_prefix: self.upload_image_to_container_registry( container_image, image_data_fh) # keep old product_id (in order to delete after update) if scan_result: product_id = scan_result.product_id() else: product_id = None try: # Upload image and update outdated analysis result with the one triggered # by the upload. scan_result = self._api.upload( application_name=self._upload_name( container_image=container_image, component=component).replace('/', '_'), group_id=self._group_id, data=image_data_fh, custom_attribs=metadata, ) finally: image_data_fh.close() for triage in triages: if triage.scope() is TriageScope.GROUP: self._api.add_triage( triage=triage, scope=TriageScope.GROUP, group_id=self._group_id, ) else: # hard-code scope for now self._api.add_triage( triage=triage, scope=TriageScope.RESULT, product_id=scan_result.product_id(), ) # rm (now outdated) scan result if product_id: self._api.delete_product(product_id=product_id) if upload_action.rescan: self._api.rescan(scan_result.product_id()) result = self._api.wait_for_scan_result( product_id=scan_result.product_id()) if result.status() == ProcessingStatus.BUSY: # Should not happen since we waited until the scan result is ready. raise RuntimeError( 'Analysis of container-image {c} was reported as completed, ' 'but is still pending'.format(c=container_image.name(), )) else: upload_status = UploadStatus.DONE return upload_result(status=upload_status, result=result)
def main(args): ec2_conf = get_ec2_conf() conn = get_conn() if args.submit: info( 'waiting for spot instance requests to be fulfilled, you can cancel by ctrl+c ...' ) try: requests = submit_request(conn, ec2_conf) except (KeyboardInterrupt, RequestFailedError) as e: error(e) exit(1) info('spot instance requests fulfilled') instance_id_to_tag_ip = {} rid_tag = request_id_to_tag(requests) info('getting instance IPs...') for r in requests: instance_id = r.instance_id info('waiting for ip to be allocated to the machine') ip = conn.get_only_instances([instance_id])[0].ip_address while ip is None: time.sleep(1) ip = conn.get_only_instances([instance_id])[0].ip_address instance_id_to_tag_ip[instance_id] = (rid_tag[r.id], ip) info('mocking vagrant info under .vagrant...') mock_vagrant_info(instance_id_to_tag_ip) info('creation of spot instances done') info('waiting for ssh to be available...') wait_for_ssh([ip for tag, ip in instance_id_to_tag_ip.values()]) info('ssh for all instances are ready') elif args.cancel: cancel_request(conn)
def _get_file_url_anonymous(self,page,post_url,headers,captcha_cb): data = util.request(self._url('reloadXapca.php')) capdata = json.loads(data) captcha = capdata['image'] if not captcha.startswith('http'): captcha = 'http:' + captcha sound = capdata['sound'] if not sound.startswith('http'): sound = 'http:' + sound # ask callback to provide captcha code self.info('Asking for captcha img %s' % captcha) code = captcha_cb({'id':captcha,'img': captcha,'snd':sound}) if not code: self.info('Captcha not provided, done') return ts = re.search('<input type=\"hidden\" name=\"ts\".+?value=\"([^\"]+)"',page,re.IGNORECASE | re.DOTALL) cid = re.search('<input type=\"hidden\" name=\"cid\".+?value=\"([^\"]+)"',page,re.IGNORECASE | re.DOTALL) sign = re.search('<input type=\"hidden\" name=\"sign\".+?value=\"([^\"]+)"',page,re.IGNORECASE | re.DOTALL) sign_a = re.search('<input type=\"hidden\" name=\"sign_a\".+?value=\"([^\"]+)"',page,re.IGNORECASE | re.DOTALL) has = capdata['hash'] salt = capdata['salt'] timestamp = capdata['timestamp'] token = re.search('<input type=\"hidden\" name=\"_token_\".+?value=\"([^\"]+)"',page,re.IGNORECASE | re.DOTALL) if not (sign and ts and cid and has and token): util.error('[uloz.to] - unable to parse required params from page, plugin needs fix') return request = { 'captcha_type':'xapca', 'hash':has, 'salt':salt, 'timestamp':timestamp, 'ts':ts.group(1), 'cid':'', 'sign':sign.group(1), 'sign_a':sign_a.group(1), 'captcha_value':code, '_do':'download-freeDownloadTab-freeDownloadForm-submit', '_token_':token.group(1), 'adi':'f' } req = urllib2.Request(post_url,urllib.urlencode(request)) req.add_header('User-Agent',util.UA) req.add_header('Referer',post_url) req.add_header('Accept','application/json') req.add_header('X-Requested-With','XMLHttpRequest') sessid=[] for cookie in re.finditer('(ULOSESSID=[^\;]+)',headers.get('Set-Cookie'),re.IGNORECASE | re.DOTALL): sessid.append(cookie.group(1)) req.add_header('Cookie','nomobile=1; uloztoid='+cid.group(1)+'; uloztoid2='+cid.group(1)+'; '+sessid[-1]) util.info(req.headers) util.info(request) try: resp = urllib2.urlopen(req) page = resp.read() headers = resp.headers except urllib2.HTTPError: # this is not OK, something went wrong traceback.print_exc() util.error('[uloz.to] cannot resolve stream url, server did not redirected us') util.info('[uloz.to] POST url:'+post_url) return try: result = json.loads(page) except: raise ResolveException('Unexpected error, addon needs fix') if not 'status' in result.keys(): raise ResolveException('Unexpected error, addon needs fix') if result['status'] == 'ok': return self._fix_stream_url(result['url']) elif result['status'] == 'error': # the only known state is wrong captcha for now util.error('Captcha validation failed, please try playing/downloading again') util.error(result) raise ResolveException('Captcha failed, try again')
def cleanUp(): info("Cleaning...") if os.path.isdir('./tmp'): shutil.rmtree('./tmp')
def validate(self): for step in self._steps: info(f"Validating step '{step.name()}'") step.validate()
def filter_and_display_upload_results( upload_results: typing.Sequence[UploadResult], cve_threshold=7, ignore_if_triaged=True, ) -> typing.Sequence[typing.Tuple[AnalysisResult, int]]: # we only require the analysis_results for now results = [r.result for r in upload_results] results_without_components = [] results_below_cve_thresh = [] results_above_cve_thresh = [] for result in results: components = result.components() if not components: results_without_components.append() continue greatest_cve = -1 for component in components: vulnerabilities = filter(lambda v: not v.historical(), component.vulnerabilities()) if ignore_if_triaged: vulnerabilities = filter(lambda v: not v.has_triage(), vulnerabilities) greatest_cve_candidate = highest_major_cve_severity( vulnerabilities) if greatest_cve_candidate > greatest_cve: greatest_cve = greatest_cve_candidate if greatest_cve >= cve_threshold: results_above_cve_thresh.append((result, greatest_cve)) continue else: results_below_cve_thresh.append((result, greatest_cve)) continue if results_without_components: warning( f'Protecode did not identify components for {len(results_without_components)}:\n' ) for result in results_without_components: print(result.display_name()) print('') def render_results_table( results: typing.Sequence[typing.Tuple[AnalysisResult, int]]): header = ('Component Name', 'Greatest CVE') results = sorted(results, key=lambda e: e[1]) result = tabulate.tabulate( map(lambda r: (r[0].display_name(), r[1]), results), headers=header, tablefmt='fancy_grid', ) print(result) if results_below_cve_thresh: info( f'The following components were below configured cve threshold {cve_threshold}' ) render_results_table(results=results_below_cve_thresh) print('') if results_above_cve_thresh: warning('The following components have critical vulnerabilities:') render_results_table(results=results_above_cve_thresh) return results_above_cve_thresh
def info(self, msg): util.info('[%s] %s' % (self.name, msg))
def estimate_radial_distortion(obj_points, img_points, K, extrinsics): util.info("Estimating radial distortion by alternation...") M = len(img_points) N = obj_points[0].shape[0] model = util.to_homogeneous_3d_multiple_points(obj_points[0]) u_c, v_c = K[0, 2], K[1, 2] # Form radius vector r = np.zeros(2 * M * N) for e, E in enumerate(extrinsics): normalized_projection = np.dot(model, E.T) normalized_projection = util.to_inhomogeneous_multiple_points( normalized_projection) x_normalized_proj, y_normalized_proj = normalized_projection[:, 0], normalized_projection[:, 1] r_i = np.sqrt(x_normalized_proj**2 + y_normalized_proj**2) r[e * N:(e + 1) * N] = r_i r[M * N:] = r[:M * N] # Form observation vector obs = np.zeros(2 * M * N) u_data, v_data = np.zeros(M * N), np.zeros(M * N) for d, data in enumerate(img_points): u_i, v_i = data[:, 0][:, 0], data[:, 0][:, 1] u_data[d * N:(d + 1) * N] = u_i v_data[d * N:(d + 1) * N] = v_i obs[:M * N] = u_data obs[M * N:] = v_data # Form prediction vector pred = np.zeros(2 * M * N) pred_centered = np.zeros(2 * M * N) u_pred, v_pred = np.zeros(M * N), np.zeros(M * N) for e, E in enumerate(extrinsics): P = np.dot(K, E) projection = np.dot(model, P.T) projection = util.to_inhomogeneous_multiple_points(projection) u_pred_i = projection[:, 0] v_pred_i = projection[:, 1] u_pred[e * N:(e + 1) * N] = u_pred_i v_pred[e * N:(e + 1) * N] = v_pred_i pred[:M * N] = u_pred pred[M * N:] = v_pred pred_centered[:M * N] = u_pred - u_c pred_centered[M * N:] = v_pred - v_c # Form distortion coefficient constraint matrix D = np.zeros((2 * M * N, 2)) D[:, 0] = pred_centered * r**2 D[:, 1] = pred_centered * r**4 # Form values (difference between sensor observations and predictions) b = obs - pred # Use pseudoinverse technique to compute least squares solution for distortion coefficients D_inv = np.linalg.pinv(D) k = np.dot(D_inv, b) util.info("DONE.") return k
def main(): n_epoch = params.n_epoch save_weight_filename = params.save_weight_file do_validation_only = params.test_only gen_n_text_samples = params.gen_n_samples learning_rate = params.learning_rate training_t = Timer() validation_t = Timer() best_pp = None prev_loss = None prev_acc = None patience = MAX_PATIENCE if params.mode == 'C2W2C': def c2w2c_weighted_objective(fn): def weighted(y_true, y_pred, weights, mask=None): assert mask is None assert weights is not None score_array = fn(y_true, y_pred) # reduce score_array to same ndim as weight array ndim = K.ndim(score_array) weight_ndim = K.ndim(weights) score_array = K.mean(score_array, axis=list(range(weight_ndim, ndim))) # apply sample weighting score_array *= weights word_scores = K.sum(score_array, axis=-1) return K.mean(word_scores) return weighted # by default Keras calculates only mean which is not correct because # word loss = sum(char losses), thus we need to monkey batch the # weighted_objective function to return correct loss for C2W2C model # ATTENTION: this might not work in later Keras versions, only tested with 1.0.5 ket.weighted_objective = c2w2c_weighted_objective # ======== PREPARE MODELS AND DATA ======== t_model, v_model, training_data, validation_data, gen_text = prepare_env( params) def validate_model(best): if gen_n_text_samples: print '\nGenerating %d text samples...' % gen_n_text_samples n_seed = 100 start = max( 0, np.random.randint(0, training_dataset.n_words - n_seed)) seed = training_dataset.get_words()[start:start + n_seed] gen_text(seed=seed, how_many=gen_n_text_samples) print '\nValidating model...' validation_t.start() v_model.set_weights(t_model.get_weights()) v_model.reset_states() n_v_samples, gen_v = validation_data[0]() loss, _ = v_model.evaluate_generator(gen_v, n_v_samples) pp = np.exp(loss) val_elapsed, val_tot = validation_t.lap() validation_info = '''Validation result: - Model loss: %f - Perplexity: %f %s - OOV rate: %f - Validation took: %s - Total validation: %s ''' % (loss, pp, delta_str(pp, best), validation_data[1], val_elapsed, val_tot) info(validation_info) return pp if do_validation_only: validate_model(None) sys.exit(0) print '\nTraining model...' for epoch in range(1, n_epoch + 1): print '=== Epoch %d ===' % epoch training_t.start() n_t_samples, gen_t = training_data[0]() t_model.reset_states() callbacks = [] if save_weight_filename: callbacks += [ ModelCheckpoint(save_weight_filename, monitor='loss', mode='min', save_best_only=True) ] h = t_model.fit_generator(generator=gen_t, samples_per_epoch=n_t_samples, callbacks=callbacks, nb_epoch=1, verbose=1) fit_elapsed, fit_tot = training_t.lap() loss = h.history['loss'][0] acc = h.history['acc'][0] epoch_info = '''Epoch %d summary at %s: - Model loss: %f %s - Model accuracy: %f %s - Perplexity: %f - Training took: %s - Total training: %s''' % ( epoch, strftime("%Y-%m-%d %H:%M:%S", localtime()), loss, delta_str(loss, prev_loss), acc, delta_str( acc, prev_acc), np.exp(loss), fit_elapsed, fit_tot) print '' info(epoch_info) pp = validate_model(best_pp) if best_pp is not None and pp > best_pp: if patience <= 0 and learning_rate > MIN_LR: learning_rate /= 2. learning_rate = max(learning_rate, MIN_LR) info( 'Validation perplexity increased. Halving learning rate to %f...\n' % learning_rate) K.set_value(t_model.optimizer.lr, learning_rate) patience = 1 else: patience -= 1 else: best_pp = pp patience = MAX_PATIENCE prev_acc = acc prev_loss = loss print 'Training complete'
def deploy_and_run_smoketest_pipeline( config_dir: str, config_name: str, concourse_team_name: str, cc_pipelines_repo_dir: str, cc_utils_repo_dir: str, wait_for_job_execution: bool=False, ): config_factory = ConfigFactory.from_cfg_dir(cfg_dir=config_dir) config_set = config_factory.cfg_set(cfg_name=config_name) concourse_cfg = config_set.concourse() # as this is an integration test, hard-code assumptions about the layout of # our pipelines repository template_path = os.path.join(cc_utils_repo_dir, 'concourse', 'templates') template_include_dir = os.path.join(cc_utils_repo_dir, 'concourse') pipeline_name = 'cc-smoketest' # retrieve pipeline-definition from github at hardcoded location github_cfg = config_set.github() githubrepobranch = GitHubRepoBranch( github_config=github_cfg, repo_owner='kubernetes', repo_name='cc-smoketest', branch='master', ) helper = GitHubRepositoryHelper.from_githubrepobranch( githubrepobranch=githubrepobranch, ) pipeline_definition = yaml.load( helper.retrieve_text_file_contents( file_path='.ci/smoketest-pipeline.yaml', ), Loader=yaml.SafeLoader, ) definition_descriptor = DefinitionDescriptor( pipeline_name=pipeline_name, pipeline_definition=pipeline_definition[pipeline_name], main_repo={'path': 'kubernetes/cc-smoketest', 'branch': 'master'}, concourse_target_cfg=concourse_cfg, concourse_target_team=concourse_team_name, ) preprocessor = DefinitionDescriptorPreprocessor() template_retriever = TemplateRetriever(template_path=template_path) renderer = Renderer( template_retriever=template_retriever, template_include_dir=template_include_dir, cfg_set=config_set, ) deployer = ConcourseDeployer( unpause_pipelines=True, expose_pipelines=True ) definition_descriptor = preprocessor.process_definition_descriptor(definition_descriptor) rendering_result = renderer.render(definition_descriptor) info('deploying pipeline') deployment_result = deployer.deploy(rendering_result.definition_descriptor) if not deployment_result.deploy_status & DeployStatus.SUCCEEDED: fail('deployment failed')
def process_results(self, results): # collect pipelines by concourse target (concourse_cfg, team_name) as key concourse_target_results = {} for result in results: definition_descriptor = result.definition_descriptor concourse_target_key = definition_descriptor.concourse_target_key() if concourse_target_key not in concourse_target_results: concourse_target_results[concourse_target_key] = set() concourse_target_results[concourse_target_key].add(result) for concourse_target_key, concourse_results in concourse_target_results.items( ): # TODO: implement eq for concourse_cfg concourse_cfg, concourse_team = next(iter( concourse_results)).definition_descriptor.concourse_target() concourse_results = concourse_target_results[concourse_target_key] concourse_api = client.from_cfg( concourse_cfg=concourse_cfg, team_name=concourse_team, ) # find pipelines to remove deployed_pipeline_names = set( map(lambda r: r.definition_descriptor.pipeline_name, concourse_results)) pipelines_to_remove = set( concourse_api.pipelines()) - deployed_pipeline_names for pipeline_name in pipelines_to_remove: info('removing pipeline: {p}'.format(p=pipeline_name)) concourse_api.delete_pipeline(pipeline_name) # trigger resource checks in new pipelines self._initialise_new_pipeline_resources(concourse_api, concourse_results) # order pipelines alphabetically pipeline_names = list(concourse_api.pipelines()) pipeline_names.sort() concourse_api.order_pipelines(pipeline_names) # evaluate results failed_descriptors = [ d for d in results if not d.deploy_status & DeployStatus.SUCCEEDED ] failed_count = len(failed_descriptors) info('Successfully replicated {d} pipeline(s)'.format(d=len(results) - failed_count)) if failed_count == 0: return True warning('Errors occurred whilst replicating {d} pipeline(s):'.format( d=failed_count, )) all_notifications_succeeded = True for failed_descriptor in failed_descriptors: warning(failed_descriptor.definition_descriptor.pipeline_name) try: self._notify_broken_definition_owners(failed_descriptor) except Exception: warning( 'an error occurred whilst trying to send error notifications' ) traceback.print_exc() all_notifications_succeeded = False # signall error only if error notifications failed return all_notifications_succeeded
def findstreams(data, regexes=None): """ Finds streams in given data. Respects caller add-on settings about quality and asks user if necessary. :param data: A string (piece of text / HTML code), an array of URLs or an array of dictionaries, where 'url' key stores actual URL and all other keys not present in item() are being copied to the resolved stream dictionary :param regexes: An array of strings - regular expressions, each MUST define named group called 'url', which retrieves resolvable URL (that one is passed to resolve operation); only used with 'data' of type 'string' :returns: An array of resolved objects, None if at least 1 resolver failed to resolve and nothing else was found, an empty array if no resolvers for URLs has been found or False if none of regexes found anything """ def get_url(obj): return obj['url'] if isinstance(obj, dict) else obj urls = [] resolvables = [] resolved = [] not_found = False if isinstance(data, str) and regexes: for regex in regexes: for match in re.finditer(regex, data, re.IGNORECASE | re.DOTALL): urls.append(match.group('url')) elif isinstance(data, list): urls = data else: raise TypeError for url in urls: if isinstance(url, dict): url['url'] = filter_resolvable(url['url']) else: url = filter_resolvable(url) if url and url not in resolvables: util.info('Found resolvable ' + get_url(url)) resolvables.append(url) if len(resolvables) == 0: util.info('No resolvables found!') return False for url in resolvables: streams = resolve(get_url(url)) if streams is None: util.info('No resolver found for ' + get_url(url)) not_found = True elif not streams: util.info('There was an error resolving ' + get_url(url)) elif len(streams) > 0: for stream in streams: if isinstance(url, dict): for key in list(url.keys()): if key not in stream: stream[key] = url[key] elif key not in item(): if isinstance(stream[key], str) and \ isinstance(url[key], str): stream[key] = url[key] + ' +' + stream[key] elif isinstance(stream[key], list) and \ isinstance(url[key], list): stream[key] = url[key] + stream[key] elif isinstance(stream[key], dict) and \ isinstance(url[key], dict): stream[key].update(url[key]) resolved.append(stream) if len(resolved) == 0: if not_found: return [] return None resolved = sorted(resolved, key=lambda i: i['quality']) resolved = sorted(resolved, key=lambda i: len(i['quality'])) resolved.reverse() return resolved
def deploy_concourse_landscape(config_name: str, deployment_name: str = 'concourse', timeout_seconds: int = '180'): ensure_not_empty(config_name) ensure_helm_setup() # Fetch all the necessary config config_factory = global_ctx().cfg_factory() config_set = config_factory.cfg_set(cfg_name=config_name) concourse_cfg = config_set.concourse() # Set the global context to the cluster specified by the given config kubernetes_config = config_set.kubernetes() kubeutil.ctx.set_kubecfg(kubernetes_config.kubeconfig()) ensure_cluster_version(kubernetes_config) # Container-registry config image_pull_secret_name = concourse_cfg.image_pull_secret() container_registry = config_factory._cfg_element( cfg_type_name='container_registry', cfg_name=image_pull_secret_name, ) cr_credentials = container_registry.credentials() # TLS config tls_config_name = concourse_cfg.tls_config() tls_config = config_factory._cfg_element(cfg_type_name='tls_config', cfg_name=tls_config_name) tls_secret_name = concourse_cfg.tls_secret_name() # Secrets server secrets_server_config = config_set.secrets_server() # Helm config helmchart_cfg_type = 'concourse_helmchart' default_helm_values = config_factory._cfg_element( cfg_type_name=helmchart_cfg_type, cfg_name=concourse_cfg.helm_chart_default_values_config()).raw custom_helm_values = config_factory._cfg_element( cfg_type_name=helmchart_cfg_type, cfg_name=concourse_cfg.helm_chart_values()).raw info('Creating default image-pull-secret ...') create_image_pull_secret( credentials=cr_credentials, image_pull_secret_name=image_pull_secret_name, namespace=deployment_name, ) info('Creating tls-secret ...') create_tls_secret( tls_config=tls_config, tls_secret_name=tls_secret_name, namespace=deployment_name, ) info('Deploying secrets-server ...') deploy_secrets_server(secrets_server_config=secrets_server_config, ) info('Deploying delaying proxy ...') deploy_delaying_proxy( concourse_cfg=concourse_cfg, deployment_name=deployment_name, ) info('Deploying Concourse ...') # Concourse is deployed last since Helm will lose connection if deployment takes more than ~60 seconds. # Helm will still continue deploying server-side, but the client will report an error. deploy_or_upgrade_concourse( default_helm_values=default_helm_values, custom_helm_values=custom_helm_values, concourse_cfg=concourse_cfg, kubernetes_config=kubernetes_config, deployment_name=deployment_name, ) info('Waiting until the webserver can be reached ...') deployment_helper = kubeutil.ctx.deployment_helper() is_web_deployment_available = deployment_helper.wait_until_deployment_available( namespace=deployment_name, name='concourse-web', timeout_seconds=timeout_seconds, ) if not is_web_deployment_available: fail( dedent("""No Concourse webserver reachable after {t} second(s). Check status of Pods created by "concourse-web"-deployment in namespace {ns} """).format( t=timeout_seconds, ns=deployment_name, )) info('Webserver became accessible.') # Even though the deployment is available, the ingress might need a few seconds to update. time.sleep(3) info('Setting teams on Concourse ...') set_teams(config=concourse_cfg)
def __init__(self): newdb.init_db() database = newdb.get_db() rows = database.execute( "SELECT value FROM {} WHERE xref_type=%(type)s AND source=%(source)s AND born<=%(now)s AND (died>%(now)s OR died is NULL)" .format(database.XREF_TABLE), { "type": database.getXrefType("TIC"), "now": util.now(), "source": database.getSourceType("compustat_idhist") }).fetchall() tickers = [ row['value'] for row in rows if re.match("[0-9].+", row["value"]) == None ] util.info("Retrieving info on %d tickers" % len(tickers)) database.close() fields = [ "symbol", "name", "exchange", "error_flag", "market_cap", "avg_daily_volume", "ex_dividend_date", "dividend_pay_date", "dividend_share_ratio", "dividend_yield", #"ebitda", "earnings_share_ratio", "eps_est_cur_year", "eps_est_next_qtr", "eps_est_next_year", "pe_ratio", "peg_ratio", "price_book_ratio", "price_eps_est_cur_year_ratio", "price_eps_est_next_year_ratio", "price_sales_ratio", "short_ratio", ] # Grab data data = ystockquote.get_symbols(tickers, fields) # Save data to temp dir tempdir = tempfile.mkdtemp(dir=os.environ['TMP_DIR']) f = open("%s/yahoo.csv" % tempdir, "w") writer = csv.DictWriter(f, fields) rows = [dict(zip(fields, fields))] rows.extend(data.values()) writer.writerows(rows) f.close() # Zip file result = os.system( "zip -j %s/yahoo-%s.csv.zip %s/yahoo.csv 1>/dev/null" % (tempdir, datetime.datetime.now().strftime("%Y%m%d%H%M"), tempdir)) if (result != 0): shutil.rmtree(tempdir) raise DataSourceError("Could not zip file") os.remove("%s/yahoo.csv" % tempdir) self._remote_dir = tempdir
#------------------------------------------------------------------------------- hardCodedProfiles = cppHelper.HardCodedProfiles(profilePath) # Supported profiles are at least composed of hard coded profiles supportedProfiles = copy.deepcopy(hardCodedProfiles.getProfilesHardCoded()) #------------------------------------------------------------------------------- xmlRootNode = xml.etree.ElementTree.parse(xmlInputFilePath).getroot() if xmlRootNode.tag != "eep": raise Exception("getAllNodes : Invalid root \"" + xmlRootNode.tag + "\", \"eep\" expected") xmlProfileNode = xmlRootNode.find("profile") util.info("Hard-coded profiles are : " + str(hardCodedProfiles.getProfilesHardCoded())) # CRorgs : Main Rorgs class, listing Rorg messages rorgsClass = cppClass.CppClass("CRorgs") rorgsItems = xmlHelper.getEnumValues(inNode=xmlProfileNode, foreachSubNode="rorg", enumValueNameTag="title", enumValueTag="number") # Add 0xD4 (Universal Teach-in message) to RORG list as it doesn't not appear in eepXX.xml file rorgsItems.append(['UTE Telegram ', '0xD4']) rorgsClass.addSubType(cppClass.CppEnumType("ERorgIds", rorgsItems, cppClass.PUBLIC)) rorgsClass.addMember(cppClass.CppMember("RorgMap", "std::map<unsigned int, std::string>", cppClass.PRIVATE, cppClass.STATIC | cppClass.CONST, \ cppHelper.getMapInitCode(rorgsItems))) rorgsClass.addMethod(cppClass.CppMethod("toRorgId", "CRorgs::ERorgIds", "unsigned int id", cppClass.PUBLIC, cppClass.STATIC, \ " if (RorgMap.find(id) == RorgMap.end())\n" \ " throw std::out_of_range(\"Unknown rorg \" + CProfileHelper::byteToHexString(id));\n" \ " return static_cast<ERorgIds>(id);\n"))
def system(self, data, cl=False): util.debug("[SC] SYSYEM CL: %s" % str(cl)) if cl is False and "setContent" in data: xbmcplugin.setContent(int(sys.argv[1]), data["setContent"]) ''' view_mode=data["setContent"].lower() skin_name=xbmc.getSkinDir() # nacitame meno skinu util.debug("[SC] skin_name='"+skin_name+"'") try: util.debug("[SC] view mode is "+view_mode) view_codes=sctop.ALL_VIEW_CODES.get(view_mode) view_code=view_codes.get(skin_name) util.debug("[SC] view code for "+view_mode+" in "+skin_name+" is "+str(view_code)) xbmc.executebuiltin("Container.SetViewMode("+str(view_code)+")") #xbmc.executebuiltin("Container.Refresh") except: util.debug("[SC] Unable to find view code for view mode "+str(view_mode)+" and skin "+skin_name) ''' if cl is False and "setPluginCategory" in data: xbmcplugin.setPluginCategory(int(sys.argv[1]), data["setPluginCategory"]) if cl is False and "addSortMethod" in data: xbmcplugin.addSortMethod( int(sys.argv[1]), sctop.sortmethod[int(data["addSortMethod"])]) if cl is False and data.get('addSortMethods'): for m in data.get("addSortMethods"): xbmcplugin.addSortMethod(int(sys.argv[1]), sctop.sortmethod[int(m)]) if cl is False and "setPluginFanart" in data: xbmcplugin.setPluginFanart(int(sys.argv[1]), data["setPluginFanart"]) if cl is False and "version" in data: util.info("[SC] kontrola verzie: %s %s" % (str(sctop.addonInfo('version')), data["version"])) if sctop.addonInfo('version') != data[ "version"] and sctop.getSetting('ver') != data['version']: try: sctop.dialog.ok( sctop.getString(30954), sctop.getString(30955) % str(data['version'])) except: pass xbmc.executebuiltin('UpdateAddonRepos') sctop.setSetting('ver', data['version']) sctop.setSetting('ws_checkssl', 'false') sctop.setSetting('check_ssl1', 'false') sctop.setSetting('usecache', 'true') if sctop.getSettingAsBool( 'cachemigrate' ) == '' or sctop.getSettingAsBool('cachemigrate') is False: self.parent.cacheMigrate() pass pass if cl is False and "focus" in data: self.parent.system = {"focus": data['focus']} if cl is True and "focus" in data: try: self.parent.endOfDirectory() util.debug("[SC] nastavujem focus na: %d" % int(data['focus'])) xel = xbmcgui.Window(xbmcgui.getCurrentWindowId()) ctr = xel.getControl(xel.getFocusId()) ctr.selectItem(int(data['focus'])) except Exception as e: util.debug("[SC] error focus :-( %s" % str(traceback.format_exc())) pass
def userData(self, all=False): self.getToken() if self.token is None: self.login() if self.token is not None: if self._userData is None: headers, req = self._create_request('/', {'wst': self.token}) try: util.info('[SC] userData') data = post(self._url('api/user_data/'), req, headers=headers, output="content") except: self.clearToken() return False util.info('[SC] userdata dat: %s' % str(data)) xml = ET.fromstring(str(data)) self._userData = xml else: xml = self._userData if not xml.find('status').text == 'OK': self.clearToken() return False if all == True: return xml util.debug("[SC] userInfo: %s %s" % (xml.find('ident').text, xml.find('vip').text)) if xml.find('vip').text == '1': xbmcgui.Window(10000).setProperty('ws.vip', '1') try: t = [ 'credits', 'points', 'files', 'bytes', 'score_files', 'score_bytes', 'private_files', 'private_bytes', 'private_space', 'tester', 'role', 'id', 'username', 'email' ] j = dict((i, xml.find(i).text) for i in t if xml.find(i) is not None) xbmcgui.Window(10000).setProperty('ws.j', str(json.dumps(j))) except Exception as e: util.debug('[SC] chyba pri natiahnuti dat %s' % str(traceback.format_exc())) try: private_space = int( xml.find('private_space').text) / 1073741824 util.info('[SC] private_space %d' % private_space) viptyp = 4 if private_space > 50 else 3 if private_space >= 50 else 2 if private_space >= 20 else 1 xbmcgui.Window(10000).setProperty('ws.viptyp', str(viptyp)) except: util.info('[SC] vip typ error %s ' % str(traceback.format_exc())) pass xbmcgui.Window(10000).setProperty('ws.ident', xml.find('ident').text) xbmcgui.Window(10000).setProperty('ws.days', xml.find('vip_days').text) return int(xml.find('vip_days').text) else: xbmcgui.Window(10000).setProperty('ws.vip', '0') return False
def teardown_test_db_app(): info("teardown_test_db_app") reset_database()
util.check_include() parser = OptionParser() parser.add_option("-d", "--debug", default=False, action="store_true", dest="debug") (options, args) = parser.parse_args() if options.debug: util.set_debug() else: util.set_log_file() util.info('launching fills listener') cfg_file = os.environ['CONFIG_DIR'] + '/exec.conf' moc_cfg_file = os.environ['CONFIG_DIR'] + '/exec.moc.conf' util.info("loading config file: %s" % cfg_file) trade_cfg = config.load_trade_config(cfg_file) moc_trade_cfg = config.load_trade_config(moc_cfg_file) gtc = guillotine.simple_multiplex_channel() gtc.connect(trade_cfg['servers'], trade_cfg['account'], trade_cfg['password'], name="fills_listener_" + os.environ['STRAT'], listenToBcast=1) gtc.connect(moc_trade_cfg['servers'], moc_trade_cfg['account'], moc_trade_cfg['password'],
def print(self, mode, loss, tictoc): info('%6s %4d:%-7d %e %s' % (mode, self.epoch_i, self.iter_i, loss, str(tictoc)))
def release_and_prepare_next_dev_cycle( githubrepobranch: GitHubRepoBranch, repository_version_file_path: str, release_version: str, repo_dir: str, release_notes_policy: str, release_commit_callback: str = None, next_version_callback: str = None, version_operation: str = "bump_minor", prerelease_suffix: str = "dev", author_name: str = "gardener-ci", author_email: str = "*****@*****.**", component_descriptor_file_path: str = None, slack_cfg_name: str = None, slack_channel: str = None, rebase_before_release: bool = False, ): release_notes_policy = ReleaseNotesPolicy(release_notes_policy) github_helper = GitHubRepositoryHelper.from_githubrepobranch( githubrepobranch) git_helper = GitHelper.from_githubrepobranch( githubrepobranch=githubrepobranch, repo_path=repo_dir, ) release_commits_step = ReleaseCommitsStep( git_helper=git_helper, repo_dir=repo_dir, release_version=release_version, repository_version_file_path=repository_version_file_path, repository_branch=githubrepobranch.branch(), version_operation=version_operation, prerelease_suffix=prerelease_suffix, release_commit_callback=release_commit_callback, next_version_callback=next_version_callback, rebase_before_release=rebase_before_release, ) github_release_step = GitHubReleaseStep( github_helper=github_helper, githubrepobranch=githubrepobranch, repo_dir=repo_dir, release_version=release_version, component_descriptor_file_path=component_descriptor_file_path, ) release_transaction = Transaction( release_commits_step, github_release_step, ) release_transaction.validate() if not release_transaction.execute(): raise RuntimeError('An error occurred while creating the Release.') publish_release_notes_step = PublishReleaseNotesStep( githubrepobranch=githubrepobranch, github_helper=github_helper, release_version=release_version, repo_dir=repo_dir, ) cleanup_draft_releases_step = CleanupDraftReleaseStep( github_helper=github_helper, release_version=release_version, ) if release_notes_policy == ReleaseNotesPolicy.DISABLED: return info('release notes were disabled - skipping') elif release_notes_policy == ReleaseNotesPolicy.DEFAULT: pass else: raise NotImplementedError(release_notes_policy) release_notes_steps = [ publish_release_notes_step, cleanup_draft_releases_step, ] if slack_cfg_name and slack_channel: post_to_slack_step = PostSlackReleaseStep( slack_cfg_name=slack_cfg_name, slack_channel=slack_channel, release_version=release_version, githubrepobranch=githubrepobranch, ) release_notes_steps.append(post_to_slack_step) release_notes_transaction = Transaction(*release_notes_steps) release_notes_transaction.validate() if not release_notes_transaction.execute(): raise RuntimeError( 'An error occurred while Publishing the Release Notes.')
def onPlayBackStarted(self): if self.scid is not None: self.onPlayBackStopped() self.upNextEnable = True self.se = None self.ep = None self.libItem = None self.watchedTime = 0 self.log("[SC] Zacalo sa prehravat") mojPlugin = self.win.getProperty(sctop.__scriptid__) if sctop.__scriptid__ not in mojPlugin: util.debug("[SC] Nieje to moj plugin ... ") return util.debug("[SC] JE to moj plugin ... %s" % str(mojPlugin)) self.scid = self.win.getProperty('scid') try: self.ids = json.loads( self.win.getProperty('%s.ids' % sctop.__scriptid__)) except: self.ids = {} pass try: stream = json.loads( self.win.getProperty('%s.stream' % sctop.__scriptid__)) util.debug("[SC] stream %s" % str(stream)) except: stream = {} pass self.stream = stream self.win.clearProperty(sctop.__scriptid__) self.win.clearProperty('%s.ids' % sctop.__scriptid__) self.win.clearProperty('%s.stream' % sctop.__scriptid__) self.win.clearProperty('scid') self.win.clearProperty('scresume') try: if sctop.getSettingAsBool('filter_audio'): util.debug("[SC] skusam vybrat spravne audio") self.selectAudio() else: util.debug("[SC] nemame filter pre audio") except: util.debug( "[SC] XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" ) pass try: if not self.isPlayingVideo(): return self.itemDuration = self.getTotalTime() # plánovaný čas dokončení 100 % přehrání self.estimateFinishTime = xbmc.getInfoLabel( 'Player.FinishTime(hh:mm:ss)') if 'originaltitle' in stream: season = stream.get('season') episode = stream.get('episode') if episode is not None and season is not None: showtitle = stream.get('originaltitle') else: showtitle = None year = stream.get('year') title = stream.get('originaltitle') try: imdb = 'tt%07d' % int(stream.get('imdb')) if stream.get( 'imdb').isdigit() else None except: imdb = None util.debug("[SC] imdb %s" % str(traceback.format_exc())) self.se = season self.ep = episode else: season = xbmc.getInfoLabel('VideoPlayer.Season') episode = xbmc.getInfoLabel('VideoPlayer.Episode') self.se = season self.ep = episode showtitle = xbmc.getInfoLabel('VideoPlayer.TVShowTitle') year = xbmc.getInfoLabel('VideoPlayer.Year') title = xbmc.getInfoLabel('VideoPlayer.Title') imdb = xbmc.getInfoLabel( "VideoPlayer.IMDBNumber") #"ListItem.IMDBNumber") if episode is not None: util.debug("[SC] Serial") self.itemType = 'episode' else: util.debug("[SC] Film") self.itemType = 'movie' try: if self.itemType == 'movie': self.addLast('lastm', self.scid) method = 'VideoLibrary.GetMovies' try: if self.ids is not None and trakt.getTraktCredentialsInfo( ) == True: #trakt.addTraktCollection({'movies':[{'ids':self.ids}]}) pass except: self.log( "[SC] trakt.tv error - nepodarilo sa pridat film do kolekcie: %s" % str(traceback.format_exc())) pass value = "%s (%s).strm" % (self.parent.normalize_filename( str(title)), str(year)) field = 'filename' res = self.executeJSON({ 'jsonrpc': '2.0', 'method': method, 'params': { 'filter': { 'operator': 'contains', 'field': field, 'value': value }, 'properties': ['file', 'resume'], }, 'id': 1 }) if 'result' in res and 'movies' in res['result']: for m in res['result']['movies']: util.debug("[SC] m: %s" % str(m)) if 'movieid' in m: self.libItem = m self.itemDBID = m['movieid'] break else: self.addLast('last', self.scid) if self.ids is not None and trakt.getTraktCredentialsInfo( ) == True: #trakt.addTraktCollection({'shows':[{'ids':self.ids}]}) pass method = 'VideoLibrary.GetTVShows' value = self.parent.normalize_filename( str(showtitle) ) #/Season %s/%sx%s.strm" % (showtitle, season, season, episode) field = 'path' res = self.executeJSON({ 'jsonrpc': '2.0', 'method': method, 'params': { 'filter': { 'operator': 'contains', 'field': field, 'value': value } }, 'id': 1 }) if 'result' in res: for m in res['result']['tvshows']: if 'tvshowid' in m: self.itemDBID = int(m['tvshowid']) res = self.executeJSON({ 'jsonrpc': '2.0', 'method': 'VideoLibrary.GetEpisodes', 'params': { 'tvshowid': int(m['tvshowid']), 'season': int(season), 'properties': ['episode', 'file', 'resume'], 'sort': { 'method': 'episode' } }, 'id': 1 }) util.info("[SC] tvshow json: %s" % str(res)) for e in res['result']['episodes']: if int(e['episode']) == int(episode): self.libItem = e self.itemDBID = e['episodeid'] break break except Exception: self.log("[SC] Chyba JSONRPC: %s" % str(traceback.format_exc())) pass res = self.executeJSON({ 'jsonrpc': '2.0', 'method': 'Player.GetItem', 'params': { 'playerid': 1 }, 'id': 1 }) if res: _filename = None try: _filename = os.path.basename(self.getPlayingFile()) except: util.debug( "[SC] onPlayBackStarted() - Exception trying to get playing filename, player suddenly stopped." ) return util.debug( "[SC] Zacalo sa prehravat: DBID: [%s], SCID: [%s] imdb: %s dur: %s est: %s fi: [%s] | %sx%s - title: %s (year: %s) showtitle: %s" % (str(self.itemDBID), str( self.scid), str(imdb), str(self.itemDuration), self.estimateFinishTime, _filename, str(season), str(episode), str(title), str(year), str(showtitle))) data = { 'scid': self.scid, 'action': 'start', 'ep': episode, 'se': season } util.debug("[SC] prehravanie %s" % str(res)) self.action(data) if 'item' in res and 'id' not in res['item']: util.debug("[SC] prehravanie mimo kniznice") except Exception: self.log("[SC] Chyba MyPlayer: %s" % str(traceback.format_exc())) pass try: if 'resume' in self.libItem: util.debug("[SC] resume! %s" % str(self.libItem)) pos = self.libItem['resume'].get('position', 0) maxPos = self.getTotalTime() * .75 if pos > 3 * 60 and pos < maxPos: self.seekTime(pos) except: pass
def action(self, data): if self.scid is None: util.debug("[SC] nemame scid") return if data.get('action', None) is None: util.debug("[SC] nemame action") return if self.stream is None: try: stream = json.loads( self.win.getProperty('%s.stream' % sctop.__scriptid__)) util.debug("[SC] stream %s" % str(stream)) self.stream = stream except: pass if data.get('action', 'None') == 'None': return url = "%s/Stats?action=%s" % (sctop.BASE_URL, data.get( 'action', 'None')) data.update({'est': self.estimateFinishTime}) data.update({'se': self.se, 'ep': self.ep}) data.update({'ver': sctop.addonInfo('version')}) try: data.update( {'state': bool(xbmc.getCondVisibility("!Player.Paused"))}) data.update({ 'ws': xbmcgui.Window(10000).getProperty('ws.ident'), 'vip': xbmcgui.Window(10000).getProperty('ws.vip'), 'vt': xbmcgui.Window(10000).getProperty('ws.viptyp'), 'j': xbmcgui.Window(10000).getProperty('ws.j') }) data.update({'vd': xbmcgui.Window(10000).getProperty('ws.days')}) data.update({'skin': xbmc.getSkinDir()}) if self.stream is not None: if 'bitrate' in self.stream: util.debug("[SC] action bitrate") data.update({'bt': self.stream['bitrate']}) else: util.debug("[SC] action no bitrate") if 'sid' in self.stream: util.info( '[SC] mame sid <====================================================================================' ) data.update({'sid': self.stream['sid']}) else: util.info( '[SC] no sid in stream <==================================================================================== %s' % str(self.stream)) except Exception as e: util.info('[SC] problem s updatom dat: %s' % str(traceback.format_exc())) pass try: if self.itemDuration > 0: data.update({'dur': self.itemDuration}) except Exception: pass lastAction = xbmcgui.Window(10000).getProperty('sc.lastAction') util.debug('[SC] lastAction: %s' % (str(lastAction))) if lastAction == "" or time() - float(lastAction) > 5: xbmcgui.Window(10000).setProperty('sc.lastAction', str(time())) util.debug("[SC] action: %s" % str(data)) url = self.parent.provider._url(url) try: sctop.post_json(url, data, {'X-UID': sctop.uid}) except: pass