def __generate_font__(self): gen_font1 = self.ui.rdoGenFont1.isChecked() temp_dir = tempfile.mkdtemp(prefix = "sdse-") temp_name = "font" font_type = font_generator.FONT_TYPES.font01 if gen_font1 else font_generator.FONT_TYPES.font02 for_game = self.ui.chkGenForGame.isChecked() for_editor = self.ui.chkGenForEditor.isChecked() font_data = self.get_font_data() if self.ui.rdoRightToLeft.isChecked(): font_data.reverse() # Add the required characters to the end of the lowest-priority font # so they're there by default but they don't override any existing # settings for them, if they're already there. font_data[-1].chars += ''.join(REQUIRED_CHARS) font = font_generator.gen_font(font_data, font_type = font_type, img_width = 1024, draw_outlines = False) font.save(temp_dir, temp_name, for_game, for_editor, font_type, game = font_generator.GAMES.sdr2) basename = os.path.join(temp_dir, temp_name) font_png = basename + ".png" font_bmp = basename + ".bmp" font_font = basename + ".font" font_dir = os.path.join(common.editor_config.data01_dir, "jp", "font", "font.pak") if for_game: game_bmp = "0000.bmp" if gen_font1 else "0002.bmp" game_font = "0001.font" if gen_font1 else "0003.font" backup_files(font_dir, [game_bmp, game_font], suffix = "_FONT") # backup_time = time.strftime("%Y.%m.%d_%H.%M.%S_FONT") # backup_dir = os.path.join(common.editor_config.backup_dir, backup_time, "font.pak") # if not os.path.isdir(backup_dir): # os.makedirs(backup_dir) # Copy our existing data into the backup directory. # shutil.copy(game_bmp, backup_dir) # shutil.copy(game_font, backup_dir) # Then replace it with the one we generated. shutil.copy(font_bmp, os.path.join(font_dir, game_bmp)) shutil.copy(font_font, os.path.join(font_dir, game_font)) if for_editor: editor_png = os.path.join(common.editor_config.gfx_dir, "font", "Font01.png" if gen_font1 else "Font02.png") editor_font = os.path.join(common.editor_config.gfx_dir, "font", "Font01.font" if gen_font1 else "Font02.font") # Copy our files in. shutil.copy(font_png, editor_png) shutil.copy(font_font, editor_font) # Reparse the font for the editor. text_printer.load_fonts() shutil.rmtree(temp_dir)
def import_umdimage(src, dst, convert_png=True, propogate=True, parent=None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError( "Cannot import %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Import Directory", "Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Any affected files will be backed up. Proceed?", buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton=QtGui.QMessageBox.No) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Finding files...", "Cancel", 0, 1, parent) progress.setWindowTitle("Importing...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) if parent: width = parent.width() height = parent.height() x = parent.x() y = parent.y() else: width = 1920 height = 1080 x = 0 y = 0 progress.setMaximum(0) progress.setValue(0) # The raw list of files we're importing. files = [] # A list of lists, including all dupes of the files being imported, too. affected_files = [] file_count = 0 dupe_base = "umdimage" tmp = tempfile.mkdtemp(prefix="sdse-") seen_groups = [] count = 0 last_update = time.time() for file in list_all_files(src): if progress.wasCanceled(): break # Strip our base directory, so we have just a relative file list. file = os.path.normpath(os.path.normcase(file[len(src) + 1:])) files.append(file) count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Finding files...\n" + file) # progress.setValue(count) progress.setValue(progress.value() ^ 1) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) affected_files.append([]) if os.path.splitext( file)[1] == ".png" and convert_png and file not in SKIP_CONV: file = os.path.splitext(file)[0] + ".gim" if propogate: file_group = _DUPE_DB.group_from_file(os.path.join( dupe_base, file)) else: file_group = None if file_group in seen_groups: continue # If there are no dupes, just add this file. if file_group == None: affected_files[-1].append(file) file_count += 1 continue seen_groups.append(file_group) for dupe in _DUPE_DB.files_in_group(file_group): # Minus the "umdimage" part dupe = dupe[len(dupe_base) + 1:] affected_files[-1].append(dupe) file_count += 1 progress.setValue(0) progress.setMaximum(file_count) # Make a backup first. backup_dir = None count = 0 for file_set in affected_files: if progress.wasCanceled(): break for file in file_set: if progress.wasCanceled(): break count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Backing up...\n" + file) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) # It's perfectly possible we want to import some files that # don't already exist. Such as when importing a directory # with added lines. if not os.path.isfile(os.path.join(dst, file)): continue backup_dir = backup_files(dst, [file], suffix="_IMPORT", backup_dir=backup_dir) progress.setValue(0) # And now do our importing. import_all_new = False skip_all_new = False count = 0 for index, src_file in enumerate(files): if progress.wasCanceled(): break if os.path.splitext(src_file)[ 1] == ".png" and convert_png and src_file not in SKIP_CONV: tmp_src_file = os.path.join(tmp, os.path.basename(src_file)) tmp_src_file = os.path.splitext(tmp_src_file)[0] + ".gim" quantize = QuantizeType.auto for regex, q in FORCE_QUANTIZE: if not regex.search(src_file) == None: quantize = q break _CONV.png_to_gim(os.path.join(src, src_file), tmp_src_file, quantize) src_file = tmp_src_file else: src_file = os.path.join(src, src_file) for file in affected_files[index]: if progress.wasCanceled(): break dst_file = os.path.join(dst, file) count += 1 # if count % 25 == 0: if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Importing...\n" + file) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) # We may be allowed to import files that don't exist, but we're # going to ask them about it anyway. if not os.path.isfile(dst_file): if skip_all_new: continue if not import_all_new: answer = QtGui.QMessageBox.question( parent, "File Not Found", "File:\n\n" + file + "\n\n" + "does not exist in the target directory. Import anyway?", buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.YesToAll | QtGui.QMessageBox.No | QtGui.QMessageBox.NoToAll, defaultButton=QtGui.QMessageBox.No) if answer == QtGui.QMessageBox.YesToAll: import_all_new = True skip_all_new = False elif answer == QtGui.QMessageBox.NoToAll: skip_all_new = True import_all_new = False continue elif answer == QtGui.QMessageBox.No: continue basedir = os.path.dirname(dst_file) if not os.path.isdir(basedir): os.makedirs(basedir) shutil.copy2(src_file, dst_file) shutil.rmtree(tmp) progress.close()
def import_umdimage2(src, dst, convert_png=True, propogate=True, parent=None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError( "Cannot import %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Import Directory", "Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Any affected files will be backed up. Proceed?", buttons=QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton=QtGui.QMessageBox.No) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Importing...", "Cancel", 0, 0, parent) progress.setWindowTitle("Importing...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) tmp_dst = tempfile.mkdtemp(prefix="sdse-") backup_dir = None for pak in glob.iglob(os.path.join(src, "bg_*.pak")): if progress.wasCanceled(): break pak_name = os.path.basename(pak) backup_dir = backup_files(dst, [pak_name], suffix="_IMPORT", backup_dir=backup_dir) # If we have a regular file with the bg_*.pak name, then just drop it in. if os.path.isfile(pak): progress.setLabelText("Importing:\n" + pak_name) progress.setValue(progress.value() ^ 1) shutil.copy2(pak, os.path.join(dst, pak_name)) # Otherwise, if it's a directory, insert all the textures we find # into the target bg_*.pak file. elif os.path.isdir(pak): for image in list_all_files(pak): if progress.wasCanceled(): break ext = os.path.splitext(image)[1].lower() if ext == ".png" and not convert_png: continue base_name = image[len(src) + 1:] dst_files = [] if propogate: dupe_name = os.path.splitext(base_name)[0] + ".gim" dupe_name = os.path.join("umdimage2", dupe_name) dupe_name = os.path.normpath(os.path.normcase(dupe_name)) dupes = _DUPE_DB.files_in_same_group(dupe_name) if dupes == None: dupes = [dupe_name] for dupe in dupes: dst_file = dupe[10:] # chop off the "umdimage2/" dst_file = os.path.splitext( dst_file)[0] + ext # original extension dst_file = os.path.join(tmp_dst, dst_file) dst_files.append(dst_file) else: dst_files = [os.path.join(tmp_dst, base_name)] for dst_file in dst_files: try: os.makedirs(os.path.dirname(dst_file)) except: pass shutil.copy(image, dst_file) if progress.wasCanceled(): break progress.setLabelText("Inserting textures into:\n" + pak_name) progress.setValue(progress.value() ^ 1) pak_dir = os.path.join(tmp_dst, pak_name) pak_file = os.path.join(dst, pak_name) # If we didn't copy anything over, just move on. if not os.path.isdir(pak_dir): continue thread = threading.Thread(target=insert_textures, args=(pak_dir, pak_file)) thread.start() while thread.isAlive(): thread.join(MIN_INTERVAL) progress.setValue(progress.value() ^ 1) if progress.wasCanceled(): progress.setLabelText("Canceling...") shutil.rmtree(tmp_dst) progress.close()
def run(self): print "Really, no need for another Steam crawler" self.request_handler.start() health_check = True start_time = time.clock() queue_time = start_time dump_time = start_time - DATA_DUMP_TIME / 2 #first dump faster dump_time2 = start_time self.session_starttime = start_time self.dump_status(1) while not self.quit or not self.request_handler.done(): if self.quit: self.request_handler.stop() if len(self.games_queue): game = self.games_queue.pop() print "Crawling game: " + game html = 1 html1 = request_html("game " + game, "http://steamcommunity.com/app/" + game) if html1[0]: self.parse_game(html1[2], game) html2 = False, else: html = self.request_handler.get_html() if html != -1: html1 = html[2] html2 = html[3] self.current_user = html[0] self.current_url = "http://steamcommunity.com/" + self.current_user if html1[0]: self.parse(html1[2], html2[2] if html2[0] else None, html[1]) #stats current_time = time.time() if html != -1 and len(html1) > 1: self.save_times.append(current_time) html2_size = html2[1] if len(html2) > 1 else 0 self.save_amounts.append(html1[1] + html2_size) self.alltimestats[1]+= 1 self.alltimestats[2]+= html1[1] + html2_size #sleep end_time = time.clock() elapsed_time = end_time - start_time sleep_time = SLEEP_TIME - elapsed_time # print time until analysis time_until_analysis = DATA_DUMP_TIME - end_time + dump_time print "Analyzing in %i:%02i \r" % (time_until_analysis // 60, time_until_analysis % 60), # sleep now if sleep_time > 0: time.sleep(sleep_time) start_time = end_time + sleep_time else: start_time = end_time #performance stats if elapsed_time < ERROR_TIME and html != -1 and len(html1) > 1: self.crawl_times_sum+= elapsed_time self.crawl_times_amount+= 1 if health_check: print "Crawling succesfully" health_check = False #data save/dump and backup if end_time - dump_time > DATA_DUMP_TIME: #sync self.request_handler.stop() if self.request_handler.done(): self.dump_data() queue_time = time.clock() dump_time = queue_time dump_time2 = queue_time self.request_handler.start() elif end_time - dump_time2 > STATUS_DUMP_TIME: #status self.dump_status() dump_time2 = end_time elif current_time > self.next_backup: #backup self.save_queue() self.next_backup = backup_files() queue_time = time.clock() elif end_time - queue_time > QUEUE_SAVE_TIME: #save self.save_queue() queue_time = time.clock() if self.quit_analyze: self.dump_data() else: self.dump_status() self.save_queue() #do this last!
def __generate_font__(self): gen_font1 = self.ui.rdoGenFont1.isChecked() temp_dir = tempfile.mkdtemp(prefix="sdse-") temp_name = "font" font_type = font_generator.FONT_TYPES.font01 if gen_font1 else font_generator.FONT_TYPES.font02 for_game = self.ui.chkGenForGame.isChecked() for_editor = self.ui.chkGenForEditor.isChecked() font_data = self.get_font_data() if self.ui.rdoRightToLeft.isChecked(): font_data.reverse() # Add the required characters to the end of the lowest-priority font # so they're there by default but they don't override any existing # settings for them, if they're already there. font_data[-1].chars += ''.join(REQUIRED_CHARS) font = font_generator.gen_font(font_data, font_type=font_type, img_width=1024, draw_outlines=False) font.save(temp_dir, temp_name, for_game, for_editor, font_type, game=font_generator.GAMES.sdr2) basename = os.path.join(temp_dir, temp_name) font_png = basename + ".png" font_bmp = basename + ".bmp" font_font = basename + ".font" font_dir = os.path.join(common.editor_config.data01_dir, "jp", "font", "font.pak") if for_game: game_bmp = "0000.bmp" if gen_font1 else "0002.bmp" game_font = "0001.font" if gen_font1 else "0003.font" backup_files(font_dir, [game_bmp, game_font], suffix="_FONT") # backup_time = time.strftime("%Y.%m.%d_%H.%M.%S_FONT") # backup_dir = os.path.join(common.editor_config.backup_dir, backup_time, "font.pak") # if not os.path.isdir(backup_dir): # os.makedirs(backup_dir) # Copy our existing data into the backup directory. # shutil.copy(game_bmp, backup_dir) # shutil.copy(game_font, backup_dir) # Then replace it with the one we generated. shutil.copy(font_bmp, os.path.join(font_dir, game_bmp)) shutil.copy(font_font, os.path.join(font_dir, game_font)) if for_editor: editor_png = os.path.join( common.editor_config.gfx_dir, "font", "Font01.png" if gen_font1 else "Font02.png") editor_font = os.path.join( common.editor_config.gfx_dir, "font", "Font01.font" if gen_font1 else "Font02.font") # Copy our files in. shutil.copy(font_png, editor_png) shutil.copy(font_font, editor_font) # Reparse the font for the editor. text_printer.load_fonts() shutil.rmtree(temp_dir)
def import_umdimage(src, dst, convert_png = True, propogate = True, parent = None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError("Cannot import %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Import Directory", "Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Any affected files will be backed up. Proceed?", buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton = QtGui.QMessageBox.No ) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Finding files...", "Cancel", 0, 1, parent) progress.setWindowTitle("Importing...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) if parent: width = parent.width() height = parent.height() x = parent.x() y = parent.y() else: width = 1920 height = 1080 x = 0 y = 0 progress.setMaximum(0) progress.setValue(0) # The raw list of files we're importing. files = [] # A list of lists, including all dupes of the files being imported, too. affected_files = [] file_count = 0 dupe_base = "umdimage" tmp = tempfile.mkdtemp(prefix = "sdse-") seen_groups = [] count = 0 last_update = time.time() for file in list_all_files(src): if progress.wasCanceled(): break # Strip our base directory, so we have just a relative file list. file = os.path.normpath(os.path.normcase(file[len(src) + 1:])) files.append(file) count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Finding files...\n" + file) # progress.setValue(count) progress.setValue(progress.value() ^ 1) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) affected_files.append([]) if os.path.splitext(file)[1] == ".png" and convert_png and file not in SKIP_CONV: file = os.path.splitext(file)[0] + ".gim" if propogate: file_group = _DUPE_DB.group_from_file(os.path.join(dupe_base, file)) else: file_group = None if file_group in seen_groups: continue # If there are no dupes, just add this file. if file_group == None: affected_files[-1].append(file) file_count += 1 continue seen_groups.append(file_group) for dupe in _DUPE_DB.files_in_group(file_group): # Minus the "umdimage" part dupe = dupe[len(dupe_base) + 1:] affected_files[-1].append(dupe) file_count += 1 progress.setValue(0) progress.setMaximum(file_count) # Make a backup first. backup_dir = None count = 0 for file_set in affected_files: if progress.wasCanceled(): break for file in file_set: if progress.wasCanceled(): break count += 1 if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Backing up...\n" + file) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) # It's perfectly possible we want to import some files that # don't already exist. Such as when importing a directory # with added lines. if not os.path.isfile(os.path.join(dst, file)): continue backup_dir = backup_files(dst, [file], suffix = "_IMPORT", backup_dir = backup_dir) progress.setValue(0) # And now do our importing. import_all_new = False skip_all_new = False count = 0 for index, src_file in enumerate(files): if progress.wasCanceled(): break if os.path.splitext(src_file)[1] == ".png" and convert_png and src_file not in SKIP_CONV: tmp_src_file = os.path.join(tmp, os.path.basename(src_file)) tmp_src_file = os.path.splitext(tmp_src_file)[0] + ".gim" quantize = QuantizeType.auto for regex, q in FORCE_QUANTIZE: if not regex.search(src_file) == None: quantize = q break _CONV.png_to_gim(os.path.join(src, src_file), tmp_src_file, quantize) src_file = tmp_src_file else: src_file = os.path.join(src, src_file) for file in affected_files[index]: if progress.wasCanceled(): break dst_file = os.path.join(dst, file) count += 1 # if count % 25 == 0: if time.time() - last_update > MIN_INTERVAL or count % 25 == 0: last_update = time.time() progress.setLabelText("Importing...\n" + file) progress.setValue(count) # Re-center the dialog. progress_w = progress.geometry().width() progress_h = progress.geometry().height() new_x = x + ((width - progress_w) / 2) new_y = y + ((height - progress_h) / 2) progress.move(new_x, new_y) # We may be allowed to import files that don't exist, but we're # going to ask them about it anyway. if not os.path.isfile(dst_file): if skip_all_new: continue if not import_all_new: answer = QtGui.QMessageBox.question( parent, "File Not Found", "File:\n\n" + file + "\n\n" + "does not exist in the target directory. Import anyway?", buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.YesToAll | QtGui.QMessageBox.No | QtGui.QMessageBox.NoToAll, defaultButton = QtGui.QMessageBox.No ) if answer == QtGui.QMessageBox.YesToAll: import_all_new = True skip_all_new = False elif answer == QtGui.QMessageBox.NoToAll: skip_all_new = True import_all_new = False continue elif answer == QtGui.QMessageBox.No: continue basedir = os.path.dirname(dst_file) if not os.path.isdir(basedir): os.makedirs(basedir) shutil.copy2(src_file, dst_file) shutil.rmtree(tmp) progress.close()
def import_umdimage2(src, dst, convert_png = True, propogate = True, parent = None): src = os.path.abspath(src) dst = os.path.abspath(dst) if os.path.normcase(src) == os.path.normcase(dst): raise ValueError("Cannot import %s. Source and destination directories are the same." % src) answer = QtGui.QMessageBox.question( parent, "Import Directory", "Importing directory:\n\n" + src + "\n\n" + "into directory:\n\n" + dst + "\n\n" + "Any affected files will be backed up. Proceed?", buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, defaultButton = QtGui.QMessageBox.No ) if answer == QtGui.QMessageBox.No: return progress = QProgressDialog("Importing...", "Cancel", 0, 0, parent) progress.setWindowTitle("Importing...") progress.setWindowModality(Qt.Qt.WindowModal) progress.setValue(0) progress.setAutoClose(False) progress.setMinimumDuration(0) tmp_dst = tempfile.mkdtemp(prefix = "sdse-") backup_dir = None for pak in glob.iglob(os.path.join(src, "bg_*.pak")): if progress.wasCanceled(): break pak_name = os.path.basename(pak) backup_dir = backup_files(dst, [pak_name], suffix = "_IMPORT", backup_dir = backup_dir) # If we have a regular file with the bg_*.pak name, then just drop it in. if os.path.isfile(pak): progress.setLabelText("Importing:\n" + pak_name) progress.setValue(progress.value() ^ 1) shutil.copy2(pak, os.path.join(dst, pak_name)) # Otherwise, if it's a directory, insert all the textures we find # into the target bg_*.pak file. elif os.path.isdir(pak): for image in list_all_files(pak): if progress.wasCanceled(): break ext = os.path.splitext(image)[1].lower() if ext == ".png" and not convert_png: continue base_name = image[len(src) + 1:] dst_files = [] if propogate: dupe_name = os.path.splitext(base_name)[0] + ".gim" dupe_name = os.path.join("umdimage2", dupe_name) dupe_name = os.path.normpath(os.path.normcase(dupe_name)) dupes = _DUPE_DB.files_in_same_group(dupe_name) if dupes == None: dupes = [dupe_name] for dupe in dupes: dst_file = dupe[10:] # chop off the "umdimage2/" dst_file = os.path.splitext(dst_file)[0] + ext # original extension dst_file = os.path.join(tmp_dst, dst_file) dst_files.append(dst_file) else: dst_files = [os.path.join(tmp_dst, base_name)] for dst_file in dst_files: try: os.makedirs(os.path.dirname(dst_file)) except: pass shutil.copy(image, dst_file) if progress.wasCanceled(): break progress.setLabelText("Inserting textures into:\n" + pak_name) progress.setValue(progress.value() ^ 1) pak_dir = os.path.join(tmp_dst, pak_name) pak_file = os.path.join(dst, pak_name) # If we didn't copy anything over, just move on. if not os.path.isdir(pak_dir): continue thread = threading.Thread(target = insert_textures, args = (pak_dir, pak_file)) thread.start() while thread.isAlive(): thread.join(MIN_INTERVAL) progress.setValue(progress.value() ^ 1) if progress.wasCanceled(): progress.setLabelText("Canceling...") shutil.rmtree(tmp_dst) progress.close()