def test_valid_email(self, email, log_file, expected_result): func_result = valid_email(email) log( log_file, f"{email} -> expected_result is {expected_result} -> actual_result is {func_result}\n" ) assert func_result == expected_result
def valid_move(string): # good example = "a1b3" if len(string) != 4: print("invalid move - string size invalid") return None # as (y, x) _from = (ord(string[0]) - ord('a'), ord(string[1]) - ord('1')) _to = (ord(string[2]) - ord('a'), ord(string[3]) - ord('1')) print("from = " + str(_from)) print("to = " + str(_to)) valid = range(0, 8) if not ((_from[0] in valid) and (_from[1] in valid)): print("invalid move - from characters invalid") return None if not ((_to[0] in valid) and (_to[1] in valid)): print("invalid move - to charaters invalid") return None ret = position_board[_from[0]][_from[1]] if game_board[_from[0]][_from[1]] == PAWN and (_to[0] == 0 or _to[0] == 7): #y is 0 or 7 diff = _from[1] - _to[1] # diff from x if diff == -1: ret += turn_queen_right elif diff == 0: ret += turn_queen_normal elif diff == 1: ret += turn_queen_left else: log("invalid turn queen ??") print("invalid turn queen ??") else: ret += position_board[_to[0]][_to[1]] return ret
def __init__(self, request, response): """ constructor """ main.log('Initializing base page.') self.initialize(request, response) self.session_store = sessions.get_store(request=request) self._initialize_login()
def run(args): detector = dlib.get_frontal_face_detector() front = frontalization.Front(args) global predictions, running, arguments predictions, running = [], True start_time = time.clock() q = [] videoDevice = ALProxy('ALVideoDevice', args.ip, args.port) captureDevice = videoDevice.subscribeCamera('Camera', 0, 2, 13, 10) image = np.zeros((480, 640, 3), np.uint8) classifier = Classifier(args, start_time, 3, args.resource_dir, [88, 88, 3], args.scope, '/'+args.model+'/') classifier.load_model() while running: image = stream2cv(image, videoDevice.getImageRemote(captureDevice)) if not image: detection = detector(image, 1) if detection: rgb_image = image[detection.top():detection.bottom(), detection.left():detection.right()] rgb_image = cv2.resize(rgb_image, (88, 88)) front_image = front.frontalized(rgb_image) prediction = classifier.classify(front_image) predictions.append(prediction) main.log(args, '\n{:.5f}s'.format(time.clock() - start_time) + ' Prediction: ' + str(prediction)) main.log(args, '\n{:.5f}s'.format(time.clock() - start_time) + ' Averages: ' + str(predictions))
def normalize(args, start_time, dirs=[]): if len(dirs) == 0: for i in [args.training_dir, args.testing_dir]: for j in ['rgb/', 'lbp/', 'frgb/', 'flbp/']: dirs.append(i + '/' + j) for dir in dirs: minimum, num_files = 1000000, [] for folder in os.listdir(dir): path = dir + '/' + folder files = len([ f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) ]) num_files.append([files, path]) main.log( args, '{:.5f}'.format(time.clock() - start_time) + 's ' + path + ' has ' + str(files) + ' files') if files < minimum: minimum = files main.log( args, '{:.5f}'.format(time.clock() - start_time) + 's minimum = ' + str(minimum)) for i in range(len(num_files)): while num_files[i][0] > minimum: os.remove(num_files[i][1] + '/' + random.choice(os.listdir(num_files[i][1]))) num_files[i][0] -= 1
def run(args): detector = dlib.get_frontal_face_detector() front = frontalization.Front(args) global predictions, running, arguments predictions, running = [], True start_time = time.clock() q = [] video = cv2.VideoCapture() classifier = Classifier(args, start_time, 3, args.resource_dir, [88, 88, 3], args.scope, '/' + args.model + '/') classifier.load_model() if video.grab(): while running: _, image = video.read() detection = detector(image, 1) if detection: rgb_image = image[detection.top():detection.bottom(), detection.left():detection.right()] rgb_image = cv2.resize(rgb_image, (88, 88)) front_image = front.frontalized(rgb_image) prediction = classifier.classify(front_image) predictions.append(prediction) main.log( args, '\n{:.5f}s'.format(time.clock() - start_time) + ' Prediction: ' + str(prediction)) main.log( args, '\n{:.5f}s'.format(time.clock() - start_time) + ' Averages: ' + str(predictions))
def handle_error(): text = "".join(cached)[:100000] if text: text = "An unexpected error has occurred:\n\n%s" % text main.log(text) report_error(text) del cached[:]
def run(args): global averages, running averages, running = True, [] start_time = time.clock() video = cv2.VideoCapture() q = [] detector = dlib.get_frontal_face_detector() front = frontalization.Front(args) classifiers = [] classifiers.append(Classifier(args, start_time, args.classes, args.resource_dir, (150,150,3), colour=True, local=False)) classifiers.append(Classifier(args, start_time, args.classes, args.resource_dir, (150, 150), colour=False, local=False)) classifiers.append(Classifier(args, start_time, args.classes, args.resource_dir, (150,150,3), colour=True, local=False)) classifiers.append(Classifier(args, start_time, args.classes, args.resource_dir, (150,150,3), colour=True, local=True)) classifiers.append(Classifier(args, start_time, args.classes, args.resource_dir, (150, 150), colour=False, local=False)) classifiers.append(Classifier(args, start_time, args.classes, args.resource_dir, (150, 150), colour=False, local=True)) if video.grab(): while running: _, frame = video.read() image = cv2.resize(frame, (150, 150, frame.shape[2])) detection = detector(image, 1) if detection: rgb_image = frame[detection.top():detection.bottom(), detection.left():detection.right()] lbp_image = feature.local_binary_pattern(cv2.cvtColor(rgb_image, cv2.COLOR_BGR2GRAY).astype(np.float64), 8, 1, 'uniform') frgb_image = front.frontalized(image) flbp_image = feature.local_binary_pattern(cv2.cvtColor(frgb_image, cv2.COLOR_BGR2GRAY).astype(np.float64), 8, 1, 'uniform') classifications = [] classifiers[0].classify(rgb_image, args.log + 'rgb') classifiers[1].classify(lbp_image, (args.log + 'lbp')) classifiers[2].classify(frgb_image, (args.log + 'frgb')) classifiers[3].classify(frgb_image, (args.log + 'lfrgb')) classifiers[4].classify(flbp_image, (args.log + 'flbp')) classifiers[5].classify(flbp_image, (args.log + 'lflbp')) result = np.zeros(args.classes) for classification in classifications: for i in range(len(classification)): result[i] += classification[i] / 6 temp = [] if len(q) < 10: q.insert(0, result) else: q.pop() q.insert(0, result) for i in range(len(q[0])): average = 0 for j in range(len(q)): average += q[j][i] / 10 temp.append(average) averages = temp main.log(args, averages) else: main.log(args, 'No Face Found', True) if sys.stdin in select.select([sys.stdin], [], [], 0)[0]: raw_input() break
def board_move(string): global game_board # y, x _from = str_to_yx(string[0]) _to = str_to_yx(string[1]) move_side = -playing_as if game_board[_from[0]][ _from[1]] < 0 else playing_as # if 1 if its the same as mine, else -1 if _to == None: if string[1] == turn_queen_normal: _to = (_from[0] - move_side, _from[1]) game_board[_from[0]][_from[1]] = QUEEN * move_side * playing_as elif string[1] == turn_queen_left: _to = (_from[0] - move_side, _from[1] - move_side) game_board[_from[0]][_from[1]] = QUEEN * move_side * playing_as elif string[1] == turn_queen_right: _to = (_from[0] - move_side, _from[1] + move_side) game_board[_from[0]][_from[1]] = QUEEN * move_side * playing_as else: log("someone choose to not take queen" + string) print("someone choose to not take queen" + string) if abs(game_board[_from[0]][_from[1]]) == KING: if abs(_to[1] - _from[1]) == 2: sgn = -1 if (_to[1] - _from[1]) < 0 else 1 if _to[1] <= 2: _from_y = 0 elif 5 <= _to[1]: _from_y = 7 board_quick_move((_to[0], _from_y), (_to[0], _to[1] - sgn)) board_quick_move(_from, _to)
def export_chats(chats, path, format, db, messages=None, skip=True, progress=None): """ Exports the specified chats from the database under path. @param chats list of chat dicts, as returned from SkypeDatabase @param path full path of directory where to save @param format export format (html|txt|xlsx|csv|filename.ext). If format is filename.ext, a single file is created: for single chat exports and multi chat XLSX exports (for multi-file exports, filenames are named by chats). @param db SkypeDatabase instance @param messages list messages to export if a single chat @param skip whether to skip chats with no messages @param progress function called before exporting each chat, with the number of messages exported so far @return (list of exported filenames, number of chats exported) """ files, count = [], 0 def make_filename(chat): if len(format) > 4: # Filename already given in format filename = os.path.join(path, format) else: args = collections.defaultdict(str); args.update(chat) filename = "%s.%s" % (conf.ExportChatTemplate % args, format) filename = os.path.join(path, util.safe_filename(filename)) filename = util.unique_path(filename) return filename main.logstatus("Exporting %s from %s %sto %s.", util.plural("chat", chats), db.filename, "" if len(format) > 4 else "as %s " % format.upper(), format if len(format) > 4 else path) if format.lower().endswith(".xlsx"): filename = make_filename(chats[0]) count = export_chats_xlsx(chats, filename, db, messages, skip, progress) files.append(filename) else: if not os.path.exists(path): os.makedirs(path) export_func = (export_chats_xlsx if format.lower().endswith("xlsx") else export_chat_csv if format.lower().endswith("csv") else export_chat_template) message_count = 0 for chat in chats: if skip and not messages and not chat["message_count"]: main.log("Skipping exporting %s: no messages.", chat["title_long_lc"]) if progress: progress(message_count) continue # continue for chat in chats main.status("Exporting %s.", chat["title_long_lc"]) if progress: progress(message_count) filename = make_filename(chat) msgs = messages or db.get_messages(chat) chatarg = [chat] if "xlsx" == format.lower() else chat export_func(chatarg, filename, db, msgs) message_count += chat["message_count"] files.append(filename) count = len(files) return (files, count)
def session(self): # Returns a session using the default cookie key. session = None if (self.session_store): session = self.session_store.get_session() else: main.log('Session store not found.') return session
def get(self): email = self.request.get('email') name = self.request.get('name') nick = self.request.get('nick') main.log('User '+ nick +' wants to login: '******'user_email'] = email self.redirect('/')
def get(self): # self.response.headers['Content-Type'] = 'text/plain' uid = self.request.get("uid",None) dim_two_level_one = self.request.get("id",None) ##### FOR LOAD TESTING ONLY ####### # generates the uid and dim2l1 id at random if not uid: uid = uuid.uuid4() if not dim_two_level_one: dim_two_level_one = DimensionTwoLevelOne.key_from_key_name("k:%02d:%02d"%(random.randint(0,9),random.randint(0,9))) # v1 = self.response.get("k1") # v2 = self.response.get("k1") logging.info("instance_id: %s"%instance_id) t = time.time() request_id = "%s:%s:%s"%(t,uid,dim_two_level_one) # 90 percent of the time we attach information to the request if random.randint(0,9) != 0: # level3:level2:level1 dim_one_level_one_key_name = "k:%s:%s:%s"%(random.randint(0,9),random.randint(0,9),random.randint(0,9)) dim_one_level_one = str(DimensionOneLevelOne.key_from_key_name(dim_one_level_one_key_name)) else: dim_one_level_one_key_name = None dim_one_level_one = None logging_params = {"uid":uid,"dim2l1":dim_two_level_one,"t":t,"req":request_id} if dim_one_level_one: imp_id = request_id + ":" + str(dim_one_level_one) logging_params.update(dim1l1=dim_one_level_one_key_name,imp=imp_id) if self.request.get("local","0") == "0": deadline = self.request.get("deadline") deadline = float(deadline) if deadline else None rpc = urlfetch.create_rpc(deadline=deadline) data = urllib.urlencode(logging_params) url = "http://eventrackerscaletest.appspot.com/log/one" url = "http://localhost:8081/log/one" logging.info("URL: %s with deadline: %s"%(url+"?"+data,deadline)) urlfetch.make_fetch_call(rpc,url+"?"+data) try: result = rpc.get_result() if result.status_code == 200: html = result.content else: html = "ERROR" except urlfetch.DownloadError: html = None html = "WHO CARES" else: log(**logging_params) html = "LOGGED" self.response.out.write('<html><head/><body><b>Hello, webapp World! %s %s %s %s <br/> %s <br/> %s </b></body></html>'%(request_id,dim_two_level_one,uid,t,logging_params, html))
def test_log(mock_datetime, mock_sensor, mock_controller): mock_sensor.value = 10 mock_controller.value = 0 t = datetime.datetime.now() mock_datetime.datetime.now.return_value = t with mock.patch("main.open", mock.mock_open()) as mock_open: main.log(mock_sensor, mock_controller) mock_open().write.assert_called_with(f"{t},10,0\n")
def train(self, training_data, testing_data): x, y = [m[0] for m in training_data], [n[1] for n in training_data] monitor = MonitorCallback(self.args, self.start_time) self.model.fit(x, y, n_epoch=self.args.epochs, validation_set=0.1, shuffle=True, show_metric=True, batch_size=self.args.batch_size, snapshot_step=2000, snapshot_epoch=True, run_id=self.args.model_name, callbacks=monitor) main.log(self.args, '{:.5f}s '.format(time.clock() - self.start_time) + str(self.count_trainable_vars()) + ' trainable parameters') self.model.save(self.save_path + 'model.model') predictions, labels = self.evaluate(testing_data) self.confusion_matrix(self.args, predictions, labels)
def run_network(args): global running try: ws = websocket.WebSocketApp(args.address, on_message=on_data, on_error=on_error, on_close=on_close) ws.on_open = on_open ws.run_forever() running = False except Exception as err: main.log(args, err.message, True)
def save_best(self, eid): data_path = os.path.join(parent_path, "training_data") folder = os.path.join(data_path, self.name + "_data") filename = "{}/epoch".format(folder) self.saver.save(self.session, filename, global_step=eid) os.unlink("%s-%d.meta" % (filename, eid)) file_name = os.path.join(parent_path, self.name + "-best-evaluation.txt") log("best meta-optimizer up to now: eid = {}".format(eid), file_name) print self.name, "saved."
def send_report(content, type, screenshot=""): """Posts feedback or error data to the report web service.""" global url_opener try: data = {"content": content.encode("utf-8"), "type": type, "screenshot": base64.b64encode(screenshot), "version": "%s-%s" % (conf.Version, get_install_type())} url_opener.open(conf.ReportURL, urllib.urlencode(data)) main.log("Sent %s report to %s (%s).", type, conf.ReportURL, content) except: main.log("Failed to send %s to %s.\n\n%s", type, conf.ReportURL, traceback.format_exc())
def updateSliders(self): power = self.main.serialGet("power?\n", 150) degrees = self.main.serialGet("degrees?\n", 150) if not (power and degrees): main.log("Error getting values") return power = int(power) degrees = int(degrees) self.horizontalSlider_power.setValue(power) self.horizontalSlider_degrees.setValue(degrees) self.label_power.setNum(power) self.label_range.setNum(degrees)
def download_and_install(url): """ Downloads and launches the specified file. """ global update_window, url_opener try: is_cancelled = False parent = wx.GetApp().TopWindow filename, tmp_dir = os.path.split(url)[-1], tempfile.mkdtemp() dlg_progress = \ controls.ProgressWindow(parent, "Downloading %s" % filename) dlg_progress.SetGaugeForegroundColour(conf.GaugeColour) dlg_progress.Position = (parent.Position.x + parent.Size.width - dlg_progress.Size.width, parent.Position.y + parent.Size.height - dlg_progress.Size.height) update_window = dlg_progress urlfile = url_opener.open(url) filepath = os.path.join(tmp_dir, filename) main.log("Downloading %s to %s.", url, filepath) filesize = int(urlfile.headers.get("content-length", sys.maxint)) with open(filepath, "wb") as f: BLOCKSIZE = 65536 bytes_downloaded = 0 buf = urlfile.read(BLOCKSIZE) while len(buf): f.write(buf) bytes_downloaded += len(buf) percent = 100 * bytes_downloaded / filesize msg = "%d%% of %s" % (percent, util.format_bytes(filesize)) is_cancelled = not dlg_progress.Update(percent, msg) if is_cancelled: break # break while len(buf) wx.YieldIfNeeded() buf = urlfile.read(BLOCKSIZE) dlg_progress.Destroy() update_window = None if is_cancelled: main.log("Upgrade cancelled, erasing temporary file %s.", filepath) util.try_until(lambda: os.unlink(filepath)) util.try_until(lambda: os.rmdir(tmp_dir)) else: main.log("Successfully downloaded %s of %s.", util.format_bytes(filesize), filename) dlg_proceed = controls.NonModalOKDialog( parent, "Update information", "Ready to open %s. You should close %s before upgrading." % (filename, conf.Title)) def proceed_handler(event): global update_window update_window = None dlg_proceed.Destroy() util.start_file(filepath) update_window = dlg_proceed dlg_proceed.Bind(wx.EVT_CLOSE, proceed_handler) except Exception: main.log("Failed to download new version from %s.\n\n%s", url, traceback.format_exc())
def export_chats_xlsx(chats, filename, db, messages=None, skip=True, progress=None): """ Exports the chats to a single XLSX file with chats on separate worksheets. @param chats list of chat data dicts, as returned from SkypeDatabase @param filename full path and filename of resulting file @param db SkypeDatabase instance @param messages list of messages to export if a single chat @param skip whether to skip chats with no messages @param progress function called before exporting each chat, with the number of messages exported so far @return number of chats exported """ count, style = 0, {0: "timestamp", 2: "wrap", 3: "hidden"} writer = xlsx_writer(filename, autowrap=[2]) message_count = 0 for chat in chats: if skip and not messages and not chat["message_count"]: main.log("Skipping exporting %s: no messages.", chat["title_long_lc"]) continue # continue for chat in chats main.status("Exporting %s.", chat["title_long_lc"]) if progress: progress(message_count) parser = skypedata.MessageParser(db, chat=chat, stats=False) writer.add_sheet(chat["title"]) writer.set_header(True) writer.writerow(["Time", "Author", "Message", "Skype Name"], {3: "boldhidden"}) writer.set_header(False) msgs = messages or db.get_messages(chat) for m in msgs: text = parser.parse(m, output={"format": "text"}) try: text = text.decode("utf-8") except UnicodeError: pass values = [m["datetime"], m["from_dispname"], text, m["author"]] style[1] = "local" if db.id == m["author"] else "remote" writer.writerow(values, style) message_count += chat["message_count"] count += 1 writer.close() return count
def _initialize_login(self): main.log('Initializing login from session.') try: login_email = self.session['user_email'] self.logged = webuser.WebUser.fromEmail(login_email) main.log('Login object built from email: ' + str(self.logged)) except KeyError: main.log('Login object NOT FOUND') self.logged = webuser.WebUser() if self.logged.is_logged: main.log('Logged in with ' + self.logged.nick) else: main.log('User not logged in')
def save_image(args, start_time, save, data, type, images=True): detector, count = dlib.get_frontal_face_detector(), 0 front = frontalization.Front(args) for image_file in data: if images: image = cv2.imread(image_file[0], cv2.IMREAD_COLOR) else: image = image_file[0] detections = detector(image, 1) for _, detection in enumerate(detections): #try: face = image[detection.top():detection.bottom(), detection.left():detection.right()] if face.shape[0] < 10 or face.shape[1] < 10: continue face = cv2.resize(face, (88, 88)) images = [] images.append(face) images.append(hue(face, 5)) images.append(hue(face, -5)) images.append(noisy('sp', images[0])) images.append(noisy('gauss', images[0])) images.append(hue(noisy('sp', images[0]), 5)) images.append(hue(noisy('sp', images[0]), -5)) images.append(hue(noisy('gauss', images[0]), 5)) images.append(hue(noisy('gauss', images[0]), -5)) for _image in images: if images: cv2.imwrite(save + '/rgb/' + str(image_file[1]) + '/' + str(count) + '.jpg', _image) else: cv2.imwrite(save + '/rgb/' + str(image_file[1]) + '/' + str(count) + '.jpg', _image) #lbp_image = feature.local_binary_pattern(cv2.cvtColor(_image, cv2.COLOR_BGR2GRAY).astype(np.float64), 8, 1, 'uniform').astype(np.uint8) #lbp_image *= (255 / lbp_image.max()) #cv2.imwrite(save + '/lbp/' + str(image_file[1]) + '/' + str(count + 1) + '.jpg', lbp_image) frgb_image = front.frontalized(image) if frgb_image is None: continue if frgb_image.shape[0] < 10 or frgb_image.shape[1] < 10: continue cv2.imwrite(save + '/frgb/' + str(image_file[1]) + '/' + str(count + 2) + '.jpg', cv2.resize(frgb_image, (88,88))) #flbp_image = feature.local_binary_pattern(cv2.cvtColor(frgb_image, cv2.COLOR_BGR2GRAY).astype(np.float64), 8, 1, 'uniform').astype(np.uint8) #flbp_image *= (255 / flbp_image.max()) #cv2.imwrite(save + '/flbp/' + str(image_file[1]) + '/' + str(count + 3) + '.jpg', flbp_image) count += 4 if count % 100 == 0: main.log(args, '{:.5f}'.format(time.clock() - start_time) + 's ' + str(count) + ' ' + type +' images extracted') #except: #pass main.log(args, str(time.clock() - start_time) + ' ' + type + ' Images Extracted')
def download_and_install(url): """ Downloads and launches the specified file. """ global update_window, url_opener try: is_cancelled = False parent = wx.GetApp().TopWindow filename, tmp_dir = os.path.split(url)[-1], tempfile.mkdtemp() dlg_progress = \ controls.ProgressWindow(parent, "Downloading %s" % filename) dlg_progress.SetGaugeForegroundColour(conf.GaugeColour) dlg_progress.Position = ( parent.Position.x + parent.Size.width - dlg_progress.Size.width, parent.Position.y + parent.Size.height - dlg_progress.Size.height) update_window = dlg_progress urlfile = url_opener.open(url) filepath = os.path.join(tmp_dir, filename) main.log("Downloading %s to %s.", url, filepath) filesize = int(urlfile.headers.get("content-length", sys.maxint)) with open(filepath, "wb") as f: BLOCKSIZE = 65536 bytes_downloaded = 0 buf = urlfile.read(BLOCKSIZE) while len(buf): f.write(buf) bytes_downloaded += len(buf) percent = 100 * bytes_downloaded / filesize msg = "%d%% of %s" % (percent, util.format_bytes(filesize)) is_cancelled = not dlg_progress.Update(percent, msg) if is_cancelled: break # break while len(buf) wx.YieldIfNeeded() buf = urlfile.read(BLOCKSIZE) dlg_progress.Destroy() update_window = None if is_cancelled: main.log("Upgrade cancelled, erasing temporary file %s.", filepath) util.try_until(lambda: os.unlink(filepath)) util.try_until(lambda: os.rmdir(tmp_dir)) else: main.log("Successfully downloaded %s of %s.", util.format_bytes(filesize), filename) dlg_proceed = controls.NonModalOKDialog(parent, "Update information", "Ready to open %s. You should close %s before upgrading." % (filename, conf.Title)) def proceed_handler(event): global update_window update_window = None dlg_proceed.Destroy() util.start_file(filepath) update_window = dlg_proceed dlg_proceed.Bind(wx.EVT_CLOSE, proceed_handler) except Exception: main.log("Failed to download new version from %s.\n\n%s", url, traceback.format_exc())
def grid_search(): # get gold rankings and store in structure with open(GOLD_RANKINGS_FILENAME) as goldFile: gold_rankings = getSystemRankings(goldFile) # make sure the necessary files are loaded main.load_the_files(opt=True) # get the normalization coefficients if NORMALIZE: swiki_max = max(main.unigram_frequencies.values()) #ilen_max = max(list(map(len, unigram_frequencies.keys()))) ilen_max = 28 # too much garbage, hard to normalize # set to 28, for "antidisestablishmentarianism" #wnet_max = max([get_wordnet_score(w) for w in unigram_frequencies]) wnet_max = 75 # for "break", given by the above expression else: swiki_max = ilen_max = wnet_max = 1.0 ilen_range = [1.0, 10.0, 100.0, 1000.0, 100000.0, 1000000.0, 10000000.0] wnet_range = [1.0, 10.0, 100.0, 1000.0, 100000.0, 1000000.0, 10000000.0] swiki_range = [1.0, 10.0, 100.0, 1000.0, 100000.0, 1000000.0, 10000000.0] cv_range = [1.0, 10.0, 100.0, 1000.0, 100000.0, 1000000.0, 10000000.0] results = Counter() i = 0 for ilen_weight in ilen_range: for wnet_weight in wnet_range: for swiki_weight in swiki_range: i += 1 log("iteration %d/%d"%(i,7*7*7)) weights = [ilen_weight/ilen_max, wnet_weight/wnet_max, swiki_weight/swiki_max, 0.0] stream = io.StringIO() main.main( weights=weights, output_stream=stream, opt=True ) stream.seek(0) system_rankings = getSystemRankings(stream) score = getScore(system_rankings, gold_rankings) results[tuple(weights)] = score print("Normalized system score for "+str(weights)+":", score) log("done") for res in results.most_common(10): print(str(res)+"\t\t"+str(results[res]))
def run(self): if self.id == 0: main.log(self.args, 'Starting ' + str(self.id)) run(self.args) main.log(self.args, 'Exiting ' + str(self.id)) elif self.id == 1: main.log(self.args, 'Starting ' + str(self.id)) run_network(self.args) main.log(self.args, 'Exiting ' + str(self.id))
def build_structure(args, start_time, classes): for folder in [args.training_dir, args.testing_dir]: if not os.path.exists(folder): os.makedirs(folder) os.makedirs(folder + '/rgb') for i in range(classes): os.makedirs(folder + '/rgb/' + str(i)) # os.makedirs(folder + '/lbp') # for i in range(classes): # os.makedirs(folder + '/lbp/' + str(i)) os.makedirs(folder + '/frgb') for i in range(classes): os.makedirs(folder + '/frgb/' + str(i)) # os.makedirs(folder + '/flbp') # for i in range(classes): # os.makedirs(folder + '/flbp/' + str(i)) main.log(args, '{:.5f}'.format(time.clock() - start_time) + 's Folder Structure Built')
def send_report(content, type, screenshot=""): """ Posts feedback or error data to the report web service. @return True on success, False on failure """ global url_opener try: data = {"content": content.encode("utf-8"), "type": type, "screenshot": base64.b64encode(screenshot), "version": "%s-%s" % (conf.Version, get_install_type())} url_opener.open(conf.ReportURL, urllib.urlencode(data)) main.log("Sent %s report to %s (%s).", type, conf.ReportURL, content) result = True except Exception: main.log("Failed to send %s to %s.\n\n%s", type, conf.ReportURL, traceback.format_exc()) result = False return result
def run(self): self._is_running = True while self._is_running: search = self._queue.get() self._stop_work = False self._drop_results = False found = {} # { Skype handle: 1, } result = {"search": search, "results": []} if search and search["handler"]: for value in search["values"]: main.log("Searching Skype contact directory for '%s'.", value) try: for user in search["handler"].search_users(value): if user.Handle not in found: result["results"].append(user) found[user.Handle] = 1 if not (self._drop_results or len(result["results"]) % conf.SearchContactsChunk): self.postback(result) result = {"search": search, "results": []} if self._stop_work: break # break for user in search["handler"].searc.. except Exception: main.log("Error searching Skype contacts:\n\n%s", traceback.format_exc()) if result["results"] and not self._drop_results: self.postback(result) result = {"search": search, "results": []} if self._stop_work: break # break for value in search["values"] if not self._drop_results: result["done"] = True self.postback(result)
def adminParam(param, state): if param == "rem": main.Clear_Old_Record_in_DB() files_utils.RemoveAllDocs() elif param == "ret": main.Clear_Old_Record_in_DB() files_utils.RemoveAllDocs() main.Insert_New_Docs() else: if state == "ignore": main.log("ignoring: " + param) main.Ignore_Document(param) print("done") elif state == "unignore": main.log("Unignoring: " + param) main.UnIgnore_Document(param) print("done") else: print("wait") return redirect("/admin.html")
def run(self): self._is_running = True while self._is_running: search = self._queue.get() self._stop_work = False self._drop_results = False found = {} # { Skype handle: 1, } result = {"search": search, "results": []} if search and search["handler"]: for i, value in enumerate(search["values"]): main.log("Searching Skype contact directory for '%s'.", value) try: for user in search["handler"].search_users(value): if user.Handle not in found: result["results"].append(user) found[user.Handle] = 1 if not (self._drop_results or len(result["results"]) % conf.SearchContactsChunk): self.postback(result) result = {"search": search, "results": []} if self._stop_work: break # break for user in search["handler"].searc.. except Exception as e: main.log("Error searching Skype contacts:\n\n%s", traceback.format_exc()) if result["results"] and not self._drop_results: self.postback(result) result = {"search": search, "results": []} if self._stop_work: break # break for i, value in enumerate(search_values) if not self._drop_results: result["done"] = True self.postback(result)
def confusion_matrix(args, predictions, labels): y_actu = np.zeros(len(labels)) for i in range(len(labels)): for j in range(len(labels[i])): if labels[i][j] == 1.00: y_actu[i] = j y_pred = np.zeros(len(predictions)) for i in range(len(predictions)): y_pred[i] = np.argmax(predictions[i]) p_labels = pd.Series(y_pred) t_labels = pd.Series(y_actu) df_confusion = pd.crosstab(t_labels, p_labels, rownames=['Actual'], colnames=['Predicted'], margins=True) main.log(args, '\nAccuracy = ' + str(accuracy_score(y_true=y_actu, y_pred=y_pred, normalize=True)) + '\n') main.log(args, df_confusion) main.log(args, ' ') main.log(args, classification_report(y_actu, y_pred))
def run(self): self._is_running = True while self._is_running: search = self._queue.get() self._stop_work = False self._drop_results = False found = {} # { Skype handle: 1, } result = {"search": search, "results": []} if search and search["handler"]: for i, value in enumerate(search["values"]): main.log("Searching Skype contact directory for '%s'.", value) for user in search["handler"].search_users(value): if user.Handle not in found: result["results"].append(user) found[user.Handle] = 1 if not (self._drop_results or len(result["results"]) % conf.ContactResultsChunk): self.postback(result) result = {"search": search, "results": []} if self._stop_work: break # break for user in search["handler"].searc.. if result["results"] and not self._drop_results: self.postback(result) result = {"search": search, "results": []} if self._stop_work: break # break for i, value in enumerate(search_values) if not self._drop_results: result["done"] = True self.postback(result)
def train(args): start_time = time.clock() #rgb_data = [retrieve_data(args.training_dir + '/rgb'), retrieve_data(args.testing_dir + '/rgb')] front_data = [ retrieve_data(args.training_dir + '/frgb'), retrieve_data(args.testing_dir + '/frgb') ] #main.log(args, '\n---------- DeXpression, No Frontalization ----------') #Dexrgb = Classifier(args, start_time, len(rgb_data[0][0][1]), args.resource_dir, rgb_data[0][0][0].shape, '/Dexrgb/', 'DeXpression') #Dexrgb.train(rgb_data[0], rgb_data[1]) main.log(args, '\n---------- DeXpression, Frontalization ----------') Dexfront = Classifier(args, start_time, len(front_data[0][0][1]), args.resource_dir, front_data[0][0][0].shape, '/Dexfront/', 'DeXpression') Dexfront.train(front_data[0], front_data[1]) #main.log(args, '\n---------- DeepFace, No Frontalization ----------') #Deeprgb = Classifier(args, start_time, len(rgb_data[0][0][1]), args.resource_dir, rgb_data[0][0][0].shape, '/Deeprgb/', 'DeepFace') #Deeprgb.train(rgb_data[0], rgb_data[1]) main.log(args, '\n---------- DeepFace, Frontalization ----------') Deepfront = Classifier(args, start_time, len(front_data[0][0][1]), args.resource_dir, front_data[0][0][0].shape, '/Deepfront/', 'DeepFace') Deepfront.train(front_data[0], front_data[1]) #main.log(args, '\n---------- Song, No Frontalization ----------') #Songrgb = Classifier(args, start_time, len(rgb_data[0][0][1]), args.resource_dir, rgb_data[0][0][0].shape, '/Songrgb/', 'Song') #Songrgb.train(rgb_data[0], rgb_data[1]) main.log(args, '\n---------- Song, Frontalization ----------') Songfront = Classifier(args, start_time, len(front_data[0][0][1]), args.resource_dir, front_data[0][0][0].shape, '/Songfront/', 'Song') Songfront.train(front_data[0], front_data[1]) main.log( args, '\n Completed Comparison in ' + str(time.clock() - start_time) + 's\n')
def queue_events(self, timeout): time.sleep(timeout) if not _paused(): new_snapshot = self._make_snapshot(self.watch.path) created, deleted, modified = self._snapshot.diff(new_snapshot) self._snapshot = new_snapshot if deleted: log("poller: delete event appeared in %s" % deleted) if created: log("poller: create event appeared in %s" % created) if modified and not(deleted or created): log("poller: modify event appeared in %s" % modified) if modified or deleted: self.queue_event(DirDeletedEvent(self.watch.path + '*')) if modified or created: self.queue_event(DirCreatedEvent(self.watch.path + '*'))
def hill_climbing_search(): log("Started at %s"%time.ctime()) # get gold rankings and store in structure with open(GOLD_RANKINGS_FILENAME) as goldFile: global gold_rankings gold_rankings = getSystemRankings(goldFile) # make sure the necessary files are loaded main.load_the_files(opt=True) # get the normalization coefficients if NORMALIZE: swiki_max = max(main.unigram_frequencies.values()) #ilen_max = max(list(map(len, unigram_frequencies.keys()))) ilen_max = 28 # too much garbage, hard to normalize # set to 28, for "antidisestablishmentarianism" #wnet_max = max([get_wordnet_score(w) for w in unigram_frequencies]) wnet_max = 75 # for "break", given by the above expression else: swiki_max = ilen_max = wnet_max = 1.0 global check_range, norm_coefficients check_range = [1.0, 10.0, 100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 10000000.0] norm_coefficients = [ilen_max, wnet_max, swiki_max, 1.0] results = Counter() old_position = (0,0,0,0) score = evaluate_solution(old_position) results[old_position] = score print("\t"+str(old_position)+":", score) i = 0; j = 1 while True: i += 1 log("%d positions done, %d calculated"%(i,j)) # look at the neighbors neighbors = get_neighbors(results, old_position) # evaluate them for pos in neighbors: j += 1 score = evaluate_solution(pos) results[pos] = score print("\t"+str(pos)+":", score) log("%d positions done, %d calculated"%(i,j)) # see if any one of them is better best = results[old_position] new_position = None for pos in neighbors: if results[pos] > best: best = results[pos] new_position = pos if new_position: old_position = new_position else: break log("done") print("best result: %s, score is %s" \ %(str(old_position), str(results[old_position]))) for res in results.most_common(10): print("position %s\t\tscore:%s\n"%(str(res[0]), str(res[1])))
def run(self): self._is_running = True # For identifying "chat:xxx" and "from:xxx" keywords query_parser = searchparser.SearchQueryParser() result = None while self._is_running: try: search = self._queue.get() if not search: continue # continue while self._is_running is_text_output = ("text" == search.get("output")) wrap_html = None # MessageParser wrap function, for HTML output if is_text_output: TEMPLATES = { "chat": templates.SEARCH_ROW_CHAT_TXT, "contact": templates.SEARCH_ROW_CONTACT_TXT, "message": templates.SEARCH_ROW_MESSAGE_TXT, "table": templates.SEARCH_ROW_TABLE_HEADER_TXT, "row": templates.SEARCH_ROW_TABLE_TXT, } wrap_b = lambda x: "**%s**" % x.group(0) output = {"format": "text"} else: TEMPLATES = { "chat": templates.SEARCH_ROW_CHAT_HTML, "contact": templates.SEARCH_ROW_CONTACT_HTML, "message": templates.SEARCH_ROW_MESSAGE_HTML, "table": templates.SEARCH_ROW_TABLE_HEADER_HTML, "row": templates.SEARCH_ROW_TABLE_HTML, } wrap_b = lambda x: "<b>%s</b>" % x.group(0) output = {"format": "html"} width = search.get("width", -1) if width > 0: dc = wx.MemoryDC() dc.SetFont(wx.Font(8, wx.FONTFAMILY_SWISS, wx.FONTSTYLE_NORMAL, wx.FONTWEIGHT_NORMAL, face=conf.HistoryFontName)) wrap_html = lambda x: wx.lib.wordwrap.wordwrap(x, width, dc) output["wrap"] = True main.log("Searching for \"%(text)s\" in %(table)s (%(db)s)." % search) self._stop_work = False self._drop_results = False parser = skypedata.MessageParser(search["db"], wrapfunc=wrap_html) # {"output": text with results, "map": link data map} # map data: {"contact:666": {"contact": {contact data}}, } result_type, result_count, count = None, 0, 0 result = {"output": "", "map": {}, "search": search, "count": 0} sql, params, match_words = query_parser.Parse(search["text"]) # Turn wildcard characters * into regex-compatible .* match_words_re = [".*".join(map(re.escape, w.split("*"))) for w in match_words] patt = "(%s)" % "|".join(match_words_re) # For replacing matching words with <b>words</b> pattern_replace = re.compile(patt, re.IGNORECASE) # Find chats with a matching title or matching participants chats = [] if search["table"] in ["conversations", "messages"]: chats = search["db"].get_conversations() chats.sort(key=lambda x: x["title"]) chat_map = {} # {chat id: {chat data}} template_chat = step.Template(TEMPLATES["chat"]) for chat in chats: chat_map[chat["id"]] = chat if "conversations" == search["table"] and match_words: title_matches = False matching_authors = [] if self.match_all(chat["title"], match_words): title_matches = True for participant in chat["participants"]: contact = participant["contact"] if contact: for n in filter(None, [contact["fullname"], contact["displayname"], contact["identity"]]): if self.match_all(n, match_words) \ and contact not in matching_authors: matching_authors.append(contact) if title_matches or matching_authors: count += 1 result_count += 1 result["output"] += template_chat.expand(locals()) key = "chat:%s" % chat["id"] result["map"][key] = {"chat": chat["id"]} if not count % conf.SearchResultsChunk \ and not self._drop_results: result["count"] = result_count self.postback(result) result = {"output": "", "map": {}, "search": search, "count": 0} if self._stop_work: break # break for chat in chats if result["output"] and not self._drop_results: result["count"] = result_count self.postback(result) result = {"output": "", "map": {}, "search": search, "count": 0} # Find contacts with a matching name if not self._stop_work and "contacts" == search["table"] \ and match_words: count = 0 contacts = search["db"].get_contacts() # Possibly more: country (ISO code, need map), birthday # (base has YYYYMMDD in integer field). match_fields = [ "displayname", "skypename", "province", "city", "pstnnumber", "phone_home", "phone_office", "phone_mobile", "homepage", "emails", "about", "mood_text", ] template_contact = step.Template(TEMPLATES["contact"]) for contact in contacts: match = False fields_filled = {} for field in match_fields: if contact[field]: val = contact[field] if self.match_all(val, match_words): match = True val = pattern_replace.sub(wrap_b, val) fields_filled[field] = val if match: count += 1 result_count += 1 result["output"] += template_contact.expand(locals()) if not (self._drop_results or count % conf.SearchResultsChunk): result["count"] = result_count self.postback(result) result = {"output": "", "map": {}, "search": search, "count": 0} if self._stop_work: break # break for contact in contacts if result["output"] and not self._drop_results: result["count"] = result_count self.postback(result) result = {"output": "", "map": {}, "search": search, "count": 0} # Find messages with a matching body if not self._stop_work and "messages" == search["table"]: template_message = step.Template(TEMPLATES["message"]) count, result_type = 0, "messages" chat_messages = {} # {chat id: [message, ]} chat_order = [] # [chat id, ] messages = search["db"].get_messages( additional_sql=sql, additional_params=params, ascending=False, use_cache=False) for m in messages: chat = chat_map.get(m["convo_id"]) body = parser.parse(m, pattern_replace if match_words else None, output) count += 1 result_count += 1 result["output"] += template_message.expand(locals()) key = "message:%s" % m["id"] result["map"][key] = {"chat": chat["id"], "message": m["id"]} if is_text_output or (not self._drop_results and not count % conf.SearchResultsChunk): result["count"] = result_count self.postback(result) result = {"output": "", "map": {}, "search": search, "count": 0} if self._stop_work or (not is_text_output and count >= conf.MaxSearchMessages): break # break for m in messages infotext = search["table"] if not self._stop_work and "all tables" == search["table"]: infotext, result_type = "", "table row" # Search over all fields of all tables. template_table = step.Template(TEMPLATES["table"]) template_row = step.Template(TEMPLATES["row"]) for table in search["db"].get_tables(): table["columns"] = search["db"].get_table_columns( table["name"]) sql, params, words = \ query_parser.Parse(search["text"], table) if not sql: continue # continue for table in search["db"].. infotext += (", " if infotext else "") + table["name"] rows = search["db"].execute(sql, params) row = rows.fetchone() if not row: continue # continue for table in search["db"].. result["output"] = template_table.expand(locals()) count = 0 while row: count += 1 result_count += 1 result["output"] += template_row.expand(locals()) key = "table:%s:%s" % (table["name"], count) result["map"][key] = {"table": table["name"], "row": row} if not count % conf.SearchResultsChunk \ and not self._drop_results: result["count"] = result_count self.postback(result) result = {"output": "", "map": {}, "search": search, "count": 0} if self._stop_work or (not is_text_output and result_count >= conf.MaxSearchTableRows): break # break while row row = rows.fetchone() if not self._drop_results: if not is_text_output: result["output"] += "</table>" result["count"] = result_count self.postback(result) result = {"output": "", "map": {}, "search": search, "count": 0} infotext += " (%s)" % util.plural("result", count) if self._stop_work or (not is_text_output and result_count >= conf.MaxSearchTableRows): break # break for table in search["db"].. single_table = ("," not in infotext) infotext = "table%s: %s" % \ ("" if single_table else "s", infotext) if not single_table: infotext += "; %s in total" % \ util.plural("result", result_count) final_text = "No matches found." if self._drop_results: result["output"] = "" if result_count: final_text = "Finished searching %s." % infotext if self._stop_work: final_text += " Stopped by user." elif "messages" == result_type and not is_text_output \ and count >= conf.MaxSearchMessages: final_text += " Stopped at %s limit %s." % \ (result_type, conf.MaxSearchMessages) elif "table row" == result_type and not is_text_output \ and count >= conf.MaxSearchTableRows: final_text += " Stopped at %s limit %s." % \ (result_type, conf.MaxSearchTableRows) result["output"] += "</table><br /><br />%s</font>" % final_text if is_text_output: result["output"] = "" result["done"] = True result["count"] = result_count self.postback(result) main.log("Search found %(count)s results." % result) except Exception as e: if not result: result = {} result["done"], result["error"] = True, traceback.format_exc() result["error_short"] = "%s: %s" % (type(e).__name__, e.message) self.postback(result)
def build_dataset(args): start_time = time.clock() if args.dataset == 'CK+': build_structure(args, start_time, 8) image_files = [] for outer_folder in os.listdir(args.data_dir): if os.path.isdir(args.data_dir): for inner_folder in os.listdir(args.data_dir + '/' + outer_folder): if os.path.isdir(args.data_dir + '/' + outer_folder + '/' + inner_folder): for input_file in os.listdir(args.data_dir + '/' + outer_folder + '/' + inner_folder): data_type = input_file.split('.')[1].lower() if not(data_type == 'png' or data_type == 'jpg' or data_type == 'tiff'): break label_file = args.label_dir + '/' + outer_folder + '/' + inner_folder + '/' + input_file[:-4] + '_emotion.txt' if os.path.isfile(label_file): read_file = open(label_file, 'r') label = int(float(read_file.readline())) for i in range(-1, -6, -1): image_file = sorted(os.listdir(args.data_dir + '/' + outer_folder + '/' + inner_folder))[i] data_type = image_file.split('.')[1].lower() if (data_type == 'png' or data_type == 'jpg' or data_type == 'tiff'): image_files.append((args.data_dir + '/' + outer_folder + '/' + inner_folder + '/' + image_file, label)) neutral_file = sorted(os.listdir(args.data_dir + '/' + outer_folder + '/' + inner_folder))[0] data_type = neutral_file.split('.')[1] if not(data_type == 'png' or data_type == 'jpg' or data_type == 'tiff'): neutral_file = sorted(os.listdir(args.data_dir + '/' + outer_folder + '/' + inner_folder))[1] image_files.append((args.data_dir + '/' + outer_folder + '/' + inner_folder + '/' + neutral_file, 0)) main.log(args, '{:.5f}'.format(time.clock() - start_time) + 's ' + str(len(image_files)) + ' Images File Collected') extract_images(args, start_time, image_files) elif args.dataset == 'KDEF': build_structure(args, start_time, 7) image_files = [] for folder in os.listdir(args.data_dir): if os.path.isdir(args.data_dir + '/' + folder): for file in os.listdir(args.data_dir + '/' + folder): data_type = file.split('.')[1].lower() if (data_type == 'png' or data_type == 'jpg' or data_type == 'tiff') and file[6] != 'F': label = 0 if file[4:6] == 'AF': label = 3 elif file[4:6] == 'AN': label = 1 elif file[4:6] == 'DI': label = 2 elif file[4:6] == 'HA': label = 4 elif file[4:6] == 'NE': label = 0 elif file[4:6] == 'SA': label = 5 elif file[4:6] == 'SU': label = 6 image_files.append((args.data_dir + '/' + folder + '/' + file, label)) main.log(args, '{:.5f}'.format(time.clock() - start_time) + 's ' + str(len(image_files)) + ' Images File Collected') extract_images(args, start_time, image_files) elif args.dataset == 'AMFED': build_structure(args, start_time, 3) images = [] for video in os.listdir(args.data_dir): if video.split('.')[1] == 'avi': filename = args.data_dir + video #print filename try: cap = skvideo.io.vread(filename) except: print 'Error with ' + filename with open(args.label_dir + '/' + video[:-4] + '-label.csv', 'rU') as csvfile: reader = csv.reader(csvfile, delimiter=',', dialect=csv.excel_tab) skip = True for row in reader: if skip: skip = False continue if float(row[1]) > 33: frame_no = float(row[0]) * 14 images.append((cap[int(frame_no)], 2)) elif float(row[3]) > 33: frame_no = float(row[0]) * 14 images.append((cap[int(frame_no)], 1)) else: frame_no = float(row[0]) * 14 images.append((cap[int(frame_no)], 0)) main.log(args, '{:.5f}'.format(time.clock() - start_time) + 's ' + str(len(images)) + ' Images File Collected') extract_images(args, start_time, images, False) else: main.log(args, 'Please specify a dataset \'--dataset\'', True) if args.split_dir != 'none': splits = split_images(args, start_time) main.log(args, '{:.5f}'.format(time.clock() - start_time) + 's ' + 'Images have been split ') if args.normalize: normalize(args, start_time) if args.split_dir != 'none': normalize(args, start_time, dirs=splits)
m["from_dispname"].encode("utf-8"), parsed_text.encode("utf-8") ] csv_writer.writerow(values) if is_html: f.write("</table>\r\n</td></tr></table>\r\n<div class='footer'>" \ "Exported with %(app)s on %(now)s.</div>\r\n" \ "</body>\r\n</html>" % header_data ) f.close() result = True except Exception, e: if f: f.close() main.log("Export cannot access %s.\n%s", filename, traceback.format_exc() ) return result def export_grid(grid, filename, title, db, sql="", table=""): """ Exports the current contents of the specified wx.Grid to file. @param grid a wx.Grid object @param filename full path and filename of resulting file, file extension .html|.csv|.sql determines file format @param title title used in HTML @param db SkypeDatabase instance @param sql the SQL query producing the grid contents, if any @param table name of the table producing the grid contents, if any
if args.block.lower() == "server": main.change_nginx_config( domain=str(args.domain), block_type="server", prev_value=args.prev_value, new_value=args.new_value, directive=args.directive, attribute=args.attribute, ) elif args.block.lower() == "upstream": main.change_nginx_config( domain=args.domain, block_type="upstream", prev_value=args.prev_value, new_value=args.new_value, attribute=args.attribute, ) main.log("Total Execution Time: {:.4f} Secounds :)".format( time.time() - start_time), "info") else: exit("cant Identify Block Name") else: main.add_subdomain(domain=args.domain, sub_name=args.new_subdomain,) # answer = args.square**2 # if args.verbose: # print("the square of {} equals {}".format(args.square, answer)) # else: # print(answer)
def performLogin(self, name, email, nick): main.log('User '+ nick +' wants to login: '******'user_email'] = email
def on_train_end(self, state): main.log( self.args, '\n{:.5f}s'.format(time.clock() - self.start) + ' Validation Loss = {:.5f}'.format(state.val_loss) + ' Validation Accuracy = ' + str(state.val_acc))
def test_logarithm(self): self.assertAlmostEqual(main.log(5), 0.6989700043360189, 'log10 fail')
import socket,thread,sys,time import main host = "" #ip of device port = 1998 #port to connect to s = socket.socket() s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) s.bind((host,port)) ##s.settimeout(20) main.log("ready for connections") print "Ready for connections" s.listen(5) while(True): try: c,addr = s.accept() c.settimeout(20) address = addr[0] main.log("Got connection from "+str(address)) print "Got a connection from ",str(address) thread.start_new_thread(main.grabConnection,(c,addr)) except Exception as e: main.log("Error for connection thread(Still running) "+str(e)) print "Error at step 1 .. "+str(e) try:
def check_newest_version(callback=None): """ Queries the Skyperious download page for available newer releases. @param callback function to call with check result, if any @result (version, url, changes) if new version up, () if up-to-date, None if query failed """ global update_window, url_opener result = () update_window = True try: main.log("Checking for new version at %s.", conf.DownloadURL) html = url_opener.open(conf.DownloadURL).read() links = re.findall("<a[^>]*\shref=['\"](.+)['\"][^>]*>", html, re.I) links = [urllib.basejoin(conf.DownloadURL, x) for x in links[:3]] if links: # Determine release types linkmap = {} # {"src": link, "x86": link, "x64": link} for link in links: link_text = link.lower() if link_text.endswith(".zip"): linkmap["src"] = link elif link_text.endswith(".exe") and "_x64" in link_text: linkmap["x64"] = link elif link_text.endswith(".exe"): linkmap["x86"] = link install_type = get_install_type() link = linkmap[install_type] # Extract version number like 1.3.2a from skyperious_1.3.2a_x64.exe version = (re.findall("(\d[\da-z.]+)", link) + [None])[0] main.log("Newest %s version is %s.", install_type, version) try: if (version != conf.Version and canonic_version(conf.Version) >= canonic_version(version)): version = None except Exception: pass if version and version != conf.Version: changes = "" try: main.log("Reading changelog from %s.", conf.ChangelogURL) html = url_opener.open(conf.ChangelogURL).read() match = re.search("<h4[^>]*>(v%s,.*)</h4\s*>" % version, html, re.I) if match: ul = html[match.end(0):html.find("</ul", match.end(0))] lis = re.findall("(<li[^>]*>(.+)</li\s*>)+", ul, re.I) items = [re.sub("<[^>]+>", "", x[1]) for x in lis] items = map(HTMLParser.HTMLParser().unescape, items) changes = "\n".join("- " + i.strip() for i in items) if changes: title = match.group(1) changes = "Changes in %s\n\n%s" % (title, changes) except Exception: main.log("Failed to read changelog.\n\n%s.", traceback.format_exc()) url = urllib.basejoin(conf.DownloadURL, link) result = (version, url, changes) except Exception: main.log("Failed to retrieve new version from %s.\n\n%s", conf.DownloadURL, traceback.format_exc()) result = None update_window = None if callback: callback(result) return result
def get(self): who = self.request.get('user') main.log('Asked to delete user: '******'/users')
def handle_error(): text = "".join(cached)[:100000] if text: main.log(text) report_error(text) cached[:] = []
def train_optimizer(task): '''Train an RNN optimizer. Args: task: A dictionary in task_list.py which specifies the model of the optimizer and the tricks and the optimizee to use when training the optimizer. ''' flags = tf.app.flags.FLAGS session = tf.get_default_session() curr_path = os.getcwd() parent_path = os.path.split(curr_path)[0] task['optimizee']['train'].build() n_steps = task['n_steps'] if 'n_steps' in task else flags.n_steps n_bptt_steps = task[ 'n_bptt_steps'] if 'n_bptt_steps' in task else flags.n_bptt_steps use_avg_loss = task['use_avg_loss'] if 'use_avg_loss' in task else False options = task['options'] if 'options' in task else {} lr_decay_name = task['lr_decay_name'] if 'lr_decay_name' in task else None assert n_steps % n_bptt_steps == 0 model = task['model'](name=task_id(), optimizee=task['optimizee']['train'], n_bptt_steps=n_bptt_steps, n_steps=n_steps, decay_step=20, lr_decay_name=lr_decay_name, decay_rate=0.96, stair_case=False, lr=task['training lr'], use_avg_loss=use_avg_loss, **options) model.prepare_train_optimizee(task['optimizee']['tests']) log_path = os.path.join(parent_path, "log") if not os.path.exists(log_path): os.makedirs(log_path) log_filename = os.path.join(log_path, model.name + "_training.txt") fig_path = os.path.join(parent_path, "training_data", model.name + "_data") if not os.path.exists(fig_path): os.makedirs(fig_path) fig_filename = os.path.join(fig_path, model.name + ".png") session.run(tf.global_variables_initializer()) eid = flags.eid if eid != 0: model.restore(flags.eid) model.bid = flags.eid * flags.n_batches file_name = os.path.join(fig_path, "loss_curve.pkl") pickle_file = open(file_name, "rb") loss_curve = pickle.load(pickle_file) pickle_file.close() train_loss = loss_curve[:eid] val_loss = loss_curve[eid:2 * eid] val_gd_loss = loss_curve[2 * eid:3 * eid] val_loss1 = loss_curve[3 * eid:4 * eid] val_gd_loss1 = loss_curve[4 * eid:5 * eid] elif eid == 0: train_loss = [] val_loss = [] val_gd_loss = [] val_loss1 = [] val_gd_loss1 = [] else: print("check eid") best_evaluation = float("inf") while flags.n_epochs == 0 or eid < flags.n_epochs: eid += 1 loss_values = [] for i in range(flags.n_batches): ret = model.train_one_iteration(n_steps) loss_value = ret['loss'] loss_values.append(loss_value) # sys.stdout.write("\r\033[K") msg = "\riteration #%d" % i msg += ": loss = %.5f avg loss = %.5f" % ( loss_value, np.mean(loss_values)) # mean loss of a batch sys.stdout.write(msg) sys.stdout.flush() # sys.stdout.write("\r\033[K") msg = "\repoch #%d" % eid msg += ": loss = %.5f" % np.mean(loss_values) log(msg, log_filename) log(str(loss_values), log_filename) train_loss.append(np.mean(loss_values)) if eid % 10 == 0: model.save(eid) test_loss_values = {} for i in range(flags.n_tests): # flag.n_tests: batch size in test for name, avg_loss_value, gd_avg_loss_value, test_loss, gd_test_loss in model.test( eid): if name not in test_loss_values: test_loss_values[name] = { 'nn': [], 'gd': [], 'nn1': [], 'gd1': [] } test_loss_values[name]['nn'].append(avg_loss_value) test_loss_values[name]['gd'].append(gd_avg_loss_value) test_loss_values[name]['nn1'].append(np.mean(test_loss)) test_loss_values[name]['gd1'].append(np.mean(gd_test_loss)) for name in test_loss_values: log( "epoch #%d test %s: loss = %.5f gd_loss = %.5f" % (eid, name, np.mean(test_loss_values[name]['nn']), np.mean(test_loss_values[name]['gd'])), log_filename) val_loss.append(np.mean(test_loss_values[name]['nn'])) val_gd_loss.append(np.mean(test_loss_values[name]['gd'])) val_loss1.append(np.mean(test_loss_values[name]['nn1'])) val_gd_loss1.append(np.mean(test_loss_values[name]['gd1'])) fig_legend = "val_gd_" + task['optimizee']['tests'][name]['gdAlgo'] if val_loss[-1] < best_evaluation: model.save_best(eid) best_evaluation = val_loss[-1] if eid % 10 == 0: file_name = os.path.join(fig_path, "loss_curve.pkl") pickle_file = open(file_name, "wb") pickle.dump( train_loss + val_loss + val_gd_loss + val_loss1 + val_gd_loss1, pickle_file) pickle_file.close() plt.figure(1) plt.plot(np.arange(1, eid + 1, 1), train_loss, "r-") plt.plot(np.arange(1, eid + 1, 1), val_loss, "b-") plt.plot(np.arange(1, eid + 1, 1), val_gd_loss, "k-") plt.plot(np.arange(1, eid + 1, 1), val_loss1, "b--") plt.plot(np.arange(1, eid + 1, 1), val_gd_loss1, "k--") plt.legend(["train", "val", fig_legend, 'val_nn', 'val_gd']) plt.xlabel("epochs") plt.ylabel("loss") plt.draw() plt.pause(0.0001) plt.savefig(fig_filename)
def export_grid(grid, filename, title, db, sql="", table=""): """ Exports the current contents of the specified wx.Grid to file. @param grid a wx.Grid object @param filename full path and filename of resulting file, file extension .html|.csv|.sql determines file format @param title title used in HTML @param db SkypeDatabase instance @param sql the SQL query producing the grid contents, if any @param table name of the table producing the grid contents, if any """ result = False f = None # @todo do with BeautifulSoup? try: if sql: # not related to is_sql sql = sql.encode("utf-8") with open(filename, "w") as f: is_html = filename.lower().endswith(".html") is_csv = filename.lower().endswith(".csv") is_sql = filename.lower().endswith(".sql") columns = \ [col["name"].encode("utf-8") for col in grid.Table.columns] main_data = { "title": title, "db": db.filename, "app": conf.Title, "now": datetime.datetime.now().strftime("%d.%m.%Y %H:%M"), } if is_html: main_data = dict([(k, escape(v)) for k,v in main_data.items()]) # Write HTML header and table header info = "<b>SQL:</b> %s" % escape(sql) if sql \ else "" f.write(GRID_HTML_HEADER % {"info": info, "title": escape(title), "db": escape(db.filename), "count": grid.NumberRows }) f.write("<tr><th>#</th><th>%s</th></tr>" \ % "</th><th>".join(columns) ) elif is_csv: # Initialize CSV writer and write header row dialect = csv.excel # Default is "," which is actually not Excel dialect.delimiter = ";" # Default is "\r\n", which causes another "\r" to be written dialect.lineterminator = "\r" csv_writer = csv.writer(f, dialect) csv_writer.writerow(["%(title)s, source: %(db)s. " \ "Exported with %(app)s on %(now)s." % main_data ]) if sql: csv_writer.writerow(["SQL: %s" \ % sql.replace("\r", " ").replace("\n", " ") ]) csv_writer.writerow(columns) elif is_sql: f.write("-- %(title)s.\n-- Source: %(db)s.\n" \ "-- Exported with %(app)s on %(now)s.\n" % main_data ) if sql: f.write("# SQL: %s\n" % sql) str_cols = ", ".join(columns) str_vals = "%(" + ")s, %(".join(columns) + ")s" if table: # Add CREATE TABLE statement. create_sql = db.tables[table.lower()]["sql"] re_sql = re.compile( "^(CREATE\s+TABLE\s+)", re.IGNORECASE | re.MULTILINE ) create_sql = re_sql.sub( lambda m: "%sIF NOT EXISTS " % m.group(1), create_sql ) f.write("%s;\n\n" % create_sql) for i in range(grid.NumberRows): data = grid.Table.GetRow(i) values = [] if is_sql: for col_name in columns: value = data[col_name] if isinstance(value, unicode): value = value.encode("utf-8") if isinstance(value, basestring): value = "\"%s\"" % value.encode("string-escape") \ .replace("\"", "\"\"") elif value is None: value = "NULL" else: value = str(value) values.append(value) stmt = u"INSERT INTO %s (%s) VALUES (" % (table, str_cols) stmt += ", ".join(values) + ");\n" f.write(stmt) else: for col_name in columns: if isinstance(data[col_name], unicode): values.append(data[col_name].encode("utf-8")) elif data[col_name] is None: values.append("") else: values.append(str(data[col_name])) if is_html: # Some values can contain HTML, need to make it safe values = map(lambda x: escape(x, utf=False), values) values.insert(0, str(i + 1)) f.write("<tr><td>%s</td></tr>" % "</td><td>".join(values)) elif is_csv: csv_writer.writerow(values) if is_html: # Write HTML footer f.write("</table>\r\n</td></tr></table>\r\n" \ "<div class='footer'>Exported with %(app)s on %(now)s." \ "</div>\r\n</body>\r\n</html>" % main_data ) f.close() result = True except Exception, e: if f: f.close() main.log("Export cannot access %s.\n%s", filename, traceback.format_exc() )