def add_mapping_to_db(): f = FileHandler('../../data/mapping.txt') for r in f: tokens = r.split('\t') rsname, gn, mn, version = tokens[0:4] mapped_count, unique_count, pos_unique_count, neg_unique_count, s_name = tokens[4:9] params = tokens[9:] rs = Raw_sample.objects.get(name=rsname) genome = Genome.objects.get(assembly_name=gn) method = Method.objects.get(method_name=mn, method_version=version) sid = transaction.savepoint() try: ''' param_set = Parameter_set(method_obj = method) param_set.save() for p in params: t = p.split(':') if len(t) == 1: t.append('') param = Parameter(param_set = param_set, param_name = t[0], param_value = t[1]) param.save() ''' param_set = add_params_to_db(params, method) mapping = Mapping(method_obj=method, genome_obj=genome, params=param_set, sample_obj=rs) mapping.save() sample = Sample(name = s_name, mapped_tags = mapped_count, unique_tags = unique_count, positive_unique_count = pos_unique_count, negative_unique_count = neg_unique_count, mapping_obj = mapping) sample.save() transaction.savepoint_commit(sid) except IntegrityError: print 'error' transaction.savepoint_rollback(sid) f.close()
class Main: def __init__(self): self.soupHandler = SoupHandler(SystemInfo.initUri) self.downloader = Downloader() self.fileHandler = FileHandler() self.path = SystemInfo.path self.name = SystemInfo.name def introduce(self): print("kocw 다운로드 프로그램.") print("대학교 자체 cdn 전용입니다.") print("url 수정은 소스코드를 직접 수정하세요.") def end(self): print("다운로드 완료 되었습니다.") def run(self): self.introduce() self.fileHandler.safeMkdir("%s/output" % self.path) self.fileHandler.safeMkdir("%s/output/%s" % (self.path, self.name)) table = self.soupHandler.findTable() parsedOnClickStrings = self.soupHandler.parseOnClick(table) self.downloader.startIteratingDownload(parsedOnClickStrings) self.end()
def add_peakcalling_to_db(): f = FileHandler("../../data/peakcalling.txt") for r in f: tokens = r.split('\t') s_name, method_name, method_version = tokens[0:3] peak_file = tokens[3] params = tokens[4:] print "a" sample = Sample.objects.get(name=s_name) method = Method.objects.get(method_name = method_name, method_version = method_version) print 'b' ''' run = models.ForeignKey(Call_peak) chrom = models.CharField(max_length=40) start = models.PositiveIntegerField(null=False) #0-based end = models.PositiveIntegerField(null=False) #1-based size = models.PositiveIntegerField(null=False) strand = models.CharField(max_length=1, choices=(('+', 'Positive'), ('-', 'Negative'), ('.', 'Unspecified'))) tag_count = models.FloatField(null=False) summit_pos_pileup = models.PositiveSmallIntegerField(null=False) #Origin summit location by peak calling method summit_val_pileup = models.FloatField(null=False) summit_pos_5 = models.PositiveSmallIntegerField() summit_val_5 = models.FloatField() p_value = models.FloatField() q_value = models.FloatField() fold = models.FloatField() ''' sid = transaction.savepoint() try: param_set = add_params_to_db(params, method) callpeak = Call_peak(sample_obj = sample, method_obj = method, params = param_set) callpeak.save() sid = transaction.savepoint() ph = FileHandler(peak_file) for p in ph: ptokens = p.split('\t') chrom, start, end, length, strand, abs_summit, pileup, pileup_5, p_val, fold, q_val, _, tc, rpkm = ptokens start = int(start) abs_summit = int(abs_summit) tc = float(tc) #print p newp = Peak(run=callpeak, chrom=chrom, start=start-1, end=end, size=length, strand=strand, tag_count=tc, summit_pos_pileup=abs_summit-1-start, summit_val_pileup = pileup, summit_pos_5 = abs_summit-1-start, summit_val_5=pileup_5, p_value = p_val, q_value = q_val, fold = fold) newp.save() transaction.savepoint_commit(sid) ph.close() except IntegrityError, msg: print msg transaction.savepoint_rollback(sid)
def do_upload(): upload = request.files.get('upload') url = request.forms.get('url') content_type = upload.content_type name, ext = os.path.splitext(upload.filename) save_path = '/tmp' upload.filename = str(abs(hash(name)))+ ext upload.save(save_path,overwrite=True) # appends upload.filename automatically fileHandler = FileHandler(5) fileId,physicalName = fileHandler.save('/tmp/'+upload.filename ,url) return physicalName
class Downloader: def __init__(self): self.path = SystemInfo.path self.name = SystemInfo.name self.videoUriFormat = SystemInfo.videoUriFormat self.fileHandler = FileHandler() self.strParser = StrParser() def startIteratingDownload(self, parsedOnclicks): count = 0 for line in parsedOnclicks: count += 1 uri = self.strParser.findSubStr(line, "'", "'") print(uri) if not SystemInfo.pdfIncluded or count % 2 == 1: code = uri.split("/")[-1] self.fileHandler.safeMkdir("%s/output/%s/video" % (self.path, self.name)) downloadUri = self.videoUriFormat % code print("%s 다운로드 중..." % (downloadUri)) res = requests.get(downloadUri) if SystemInfo.pdfIncluded: number = (count + 1) // 2 self.fileHandler.saveBinaryFile(res.content, "video", number, "mp4") else: print("%s 다운로드 중..." % uri) res = requests.get(uri) self.fileHandler.safeMkdir("%s/output/%s/pdf" % (self.path, self.name)) number = (count + 1) // 2 self.fileHandler.saveBinaryFile(res.content, "pdf", number, "pdf")
def main(): if len(sys.argv) != 1 + 1: print("Provide the android PhotoScan file folder as argument.") return directory = sys.argv[1] handler = FileHandler() w = FileWatcher(directory, handler.processNewFile) w.run() try: while True: seriesName = input("Current series name: ") seriesDate = input("Series Date (YYYY:MM) : ") handler.setSeriesName(seriesName) handler.setSeriesDate(seriesDate) print("Running series...") # command loop command = "" while command != "n": command = input("[d]elete last image or [n]ext series? ") if command == "d": handler.deleteLastFile() except KeyboardInterrupt: print("") # force newline in terminal w.stop() sys.exit()
def add_method_to_db(): f = FileHandler("../../data/methods.txt") sid = transaction.savepoint() for r in f: tokens = r.split('\t') m = Method(method_name=tokens[0], method_type=tokens[1], method_version=tokens[2]) sid = transaction.savepoint() try: m.save() transaction.savepoint_commit(sid) except DatabaseError: transaction.savepoint_rollback(sid) f.close()
def add_genome_to_db(): f = FileHandler("../../data/genomes.txt") #sid = transaction.savepoint() for r in f: tokens = r.split('\t') g = Genome(assembly_name = tokens[0], alternative_assembly_name = tokens[1], species = tokens[2], size = tokens[3]) sid = transaction.savepoint() #create save points before transaction for rollback if error occurs. try: g.save() transaction.savepoint_commit(sid) except IntegrityError: print "error" transaction.savepoint_rollback(sid) f.close()
def add_chromosomes_to_db(): genomes = Genome.objects.all() if len(genomes) <= 0: print "No genomes. Abort." else: for g in genomes: f = FileHandler('../../data/%s.chrom.sizes'%g.assembly_name) #sid = transaction.savepoint() for r in f: cname, csize = r.split() chrom = Chromosome(name=cname, length=csize, genome_obj=g) sid = transaction.savepoint() try: chrom.save() transaction.savepoint_commit(sid) except IntegrityError: transaction.savepoint_rollback(sid) print 'error' f.close()
def make_image_sequence(input_file, output_folder, prefix='man_seg_000_'): ''' the fiji 'image sequence' can lead to the error :'Incorrect count for "ColorMap" when using Tiffreader, therefore made my own ''' filehandler = FileHandler() filehandler.d_load_info['f_name'] = str(input_file) filehandler.d_save_info['save_dir'] = str(output_folder) a_img = filehandler.load_tif() for ix_z, a_slice in enumerate(a_img): suffix = "{:03d}".format(ix_z) filehandler.d_save_info['f_name'] = "{0}{1}".format(prefix, suffix) filehandler.save_data(a_slice) return
def count(self, most_common_words, *args): """ :param dir_path: String - Full path to directory where files reside :param most_common_words: Integer - Number of most common words in all files :return: Integer - Number of most common words """ _counter = Counter() self.most_common_words = most_common_words _root_dir = DirectoryHandler() for path in map(abspath, args): if isfile(path): _counter += FileHandler().file_word_count(filename=path) else: with ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor: future_file_counts = {executor.submit(FileHandler().file_word_count, _filename): _filename for _filename in _root_dir.scan_dir(path)} for future in as_completed(future_file_counts): _counter += future.result() self.logger.debug("size of counter is {0} MB".format(round(getsizeof(_counter) / 1024 / 1024), 3)) return _counter.most_common(self.most_common_words)
class Chatbot: fh = FileHandler('newconversa.txt') def __init__(self, nome): self.nome = nome resp = self.intro() if resp.upper() == 'S': while self.conversar().lower() != '.sair': self.conversar() else: self.tchau() def intro(self): print("{}: Olá meu nome é {}".format(self.nome, self.nome)) print("{}: Eu sou uma I.A. que irá aprender com você".format( self.nome)) resp = input("{}: Vamos começar? (S/N): ".format(self.nome)) return resp def conversar(self): r = input("Você: ") if r == '.sair': return r if self.fh.comunica(r) != 0: print(self.nome + ": " + self.fh.comunica(r)) else: self.aprender(r) return r def aprender(self, p): print(self.nome + ": Não sei responder!") r = input(self.nome + ": Qual seria a resposta para essa pergunta? ") self.fh.gravarArquivo(p, r) def tchau(self): print("{}: Tchau".format(self.nome))
class TestHander(unittest.TestCase): dbHandler = None; fileHandler = None; def setUp(self): self.dbHandler = DBHandler() self.fileHandler = FileHandler(5) def testFileHanderSave(self,url = 'aa'): path = '/tmp/license.txt' import os if not os.path.isfile(path): os.mknod(path) fileId,physicalName = self.fileHandler.save(path,url) return fileId,physicalName def testGetFilePathByPhysicalName(self): fileId,physicalName = self.testFileHanderSave() path = self.fileHandler.getFilePathByDiskName(physicalName) print 'path :'+ path def testGetFiletypeByURL(self): url = 'http://www.abc.com' self.testFileHanderSave(url) print 'file type is :' + self.fileHandler.getTypeByUrl(url) def testGetFileByID(self): fileId,physicalName = self.testFileHanderSave() opendFile = self.fileHandler.getFileByID(fileId) def testGetFiletypeByID(self): fileId,physicalName = self.testFileHanderSave() print self.fileHandler.getTypeByID(fileId) def testGetTotalFileCount(self): num = self.fileHandler.getTotalFileCount() print "file count : " + str(num) def testExecute(self): sql = 'select * from files' rows = self.dbHandler.excute(sql)
from watchdog.observers import Observer import os import json import time from fileHandler import FileHandler tracked_folder = input('Type the path to the folder to be tracked: \n') destination_folder = input('Type the path to the destination folder: \n') event_handler = FileHandler(tracked_folder, destination_folder) observer = Observer() observer.schedule(event_handler, tracked_folder, recursive=True) observer.start() try: while True: time.sleep(10) except KeyboardInterrupt: observer.stop() observer.join()
def setUp(self): self.dbHandler = DBHandler() self.fileHandler = FileHandler(5)
def __init__(self): self.path = SystemInfo.path self.name = SystemInfo.name self.videoUriFormat = SystemInfo.videoUriFormat self.fileHandler = FileHandler() self.strParser = StrParser()
if __name__ == "__main__": ''' This script receive a name of an actor or actress and print the information of all the films where he/she has appear. Only full names are supported as they appear in the IMDB database, but if the name is not found, a broad spectrum search is performed. In this search, the name is splitted in all the parts it have and print the info of all the actors that fit to any of those partial names. ''' option = "" while(option != "0"): print(("Please write the name of an actor or actress to know all the films where he/she appear " + "(Only full names supported)" + " , write 0 to exit:")) option = str(input('Enter the actor name option: ')) db = FileHandler() writer = Writer() db.openActDB() if option != "0" and option != "": films = db.searchActor(option) if len(films) > 0: writer.printFilmsDataFromActor(option, films) else: print(("%s not found") % (option)) print(("A broad spectrum search will take place, wait a moment")) result = db.broadSpectrumSearchActor(option) print(("All the actors whose name contains any of the words provided will be listed now")) for key in result.keys(): writer.printFilmsDataFromActor(key, result[key])
def initialize_filehandler(): filehandler = FileHandler() filehandler.d_save_info['save_dir'] = OUTPUT_FOLDER_ROOT #print("DEBUG1: filehandler {0}, OUTPUT_FOLDER_ROOT {1}, filehandler.get_save_location() {2}".format(filehandler.__repr__(), OUTPUT_FOLDER_ROOT,filehandler.get_save_location())) if ic_timestamp_subfolder: if IMG_RAW_FILE: filehandler.d_save_info['sub_dir_1'] = str( datetime.datetime.now()).replace(":", "_").replace( " ", "_") + "_" + IMG_RAW_FILE[0:-4] else: filehandler.d_save_info['sub_dir_1'] = str( datetime.datetime.now()).replace(":", "_").replace(" ", "_") filehandler.set_save_info_root() #print("DEBUG2: filehandler {0}, OUTPUT_FOLDER_ROOT {1}, filehandler.get_save_location() {2}".format(filehandler.__repr__(), OUTPUT_FOLDER_ROOT,filehandler.get_save_location())) filehandler.d_load_info['load_dir'] = IMG_RAW_DIR filehandler.take_snapshot() #print("DEBUG3: filehandler {0}, OUTPUT_FOLDER_ROOT {1}, filehandler.get_save_location() {2}".format(filehandler.__repr__(), OUTPUT_FOLDER_ROOT,filehandler.get_save_location())) print('SDT_MAIN.py:all info will be written to : ', filehandler.get_save_location()) return filehandler
def __init__(self): self.soupHandler = SoupHandler(SystemInfo.initUri) self.downloader = Downloader() self.fileHandler = FileHandler() self.path = SystemInfo.path self.name = SystemInfo.name
''' dw = DownloadFromWeb("ftp.fu-berlin.de","pub/misc/movies/database" ) download = False option = 0 try: option = int(input("Do you want to download the new version of the databases if available? (1=yes, 0= no)")) if option != 0 and option != 1: raise ValueError() except ValueError: print("Option not found, assuming default value: no") if option == 1: download = True result = dw.fetch("actresses.list.gz", download) print(result) result = dw.fetch("actors.list.gz", download) print(result) print("All fetched") print("Loading actors") actorHandler = FileHandler() actorHandler.open("actors.list.gz") actorHandler.loadData() print("Actors loaded, closing files") actorHandler.close() print("Loading actresses") actressHandler = FileHandler() actressHandler.open("actresses.list.gz") actressHandler.loadData() print("Actresses loaded, closing files") actressHandler.close()
#CHECKS Stack.check_memory_allocation(verbose=prm['verbose']) #<<<end of stack loop finish_last_stack(Stack.curr_stack) if prm['DBG']: append_summary_of_4Dcell_txt_repr() append_labelling_chronology_to_txt_log() feat_tracker.write_excel_summary_4D_v2(fh) # Stack.save_4D_RGBA_clusterview(fh,nb_channels=1,extra_channel=get_extra_channel()) Stack.save_4D_RGBA_clusterview(fh,nb_channels=1) Stack.agg_temp_viz_files(fh,search_string='raw_clusterview') Stack.agg_temp_viz_files(fh,search_string='labelview') #Stack.save_4D_clusterview(fh) Stack.init_class_variables() if prm['DBG']:print('{0} datamodel inconsistencies found across all stacks'.format(nb_inconsistencies)) remove_temp_files() toc = time.time() if prm['verbose']:print('runtime_spheresDT = ', toc-tic) return if __name__ == '__main__': param_xml = read_parms() fh = FileHandler.init_fh(param_xml.prm) spheres_DT(param_xml,fh)
if __name__ == "__main__": ''' This script receive 2 actors o actresses names and the maximum distance between them and return the minimum number of relation (tipe "work with") between them. Remember that the higher distance, the longer the algorithm will last (default value is 11 and the minimum is 3). ''' option = "" while(option != "0"): print(("Please write the name of two actors/actresses (Only full names supported)" + " to see if they are related , write 0 to exit:")) option = str(input('Enter the actor/actress name: ')) if option != "" or option != "0": option2 = str(input('Enter the actor/actress name: ')) if option2 != "" or option2 != "0": db = FileHandler() db.openActDB() db.openFilmsDB() try: option3 = int(input('Enter maximum length of path to follow, the higher number, the longer it takes (minimum 3, default 11):')) except: option3 = 11 if option3 < 3: option3 = 3 shortestPath = ShortestPath(db, limit=option3) result = shortestPath.findPath(origin=option, destiny=option2) if result is None: print("One of the actor/actress names doesn't exists.") elif type(result) is int and result == -1: print("No relation found within " + option3 + " steps") elif type(result) is Node:
GNU General Public License for more details. You should have received a copy of the GNU General Public License along with coprotagositas. If not, see <http://www.gnu.org/licenses/>. ''' from fileHandler import FileHandler from writer import Writer if __name__ == "__main__": ''' This script recieve 2 actors or actresses and print the films where they are coprotagonist ''' option = "" while(option != "0"): print(("Please write the name of two actors/actresses to find the films they made together " + "(Only full names supported)" + " , write 0 to exit:")) option1 = str(input('Enter the actor/actress name: ')) if option1 != "" and option1 != "0": option2 = str(input('Enter the actor/actress name: ')) if option2 != "" and option2 != "0": db = FileHandler() writer = Writer() db.openActDB() resultsdata = db.searchFilms2actors(option1, option2) writer.printFilms2Actors(option1, option2, resultsdata) db.closeActDB() else: print("Exiting") else: print("Exiting")
def calculate_CTC_SEG_slice_detail(a_gt, a_res, output_folder, ix_z="", verbose=False, output=sys.stdout, d_d_gt_res_overlap={}): ''' calculate SEG score for 2 slices, with subcalculations ''' if output_folder: # f_name = output_folder / "CTC_SEG_detail_calculations.txt" # f_output = open( f_name ,"w" ) fh = FileHandler() fh.d_save_info['save_dir'] = str(output_folder) a_gt_labels = np.unique(a_gt) a_gt_labels = np.delete(a_gt_labels, [0]) if verbose: print('ground truth labels are ', a_gt_labels, file=output) SEG_counter = 0 SEG_sum = 0 jaccard_sum = 0 for gt_label in a_gt_labels: a_bool_gt = a_gt == gt_label nb_gt_label_pix = np.sum(a_bool_gt) a_res_overlap = np.where(a_bool_gt, a_res, 0) a_bincount_overlap = np.bincount(a_res_overlap.flatten()) a_bincount_overlap[0] = 0 a_res_labels_overlap = np.unique(a_res_overlap) a_res_labels_overlap = np.delete(a_res_labels_overlap, [0]) if verbose: print(' ground truth label {0} overlaps with res labels {1} '. format(gt_label, a_res_labels_overlap), file=output) max_overlap_label = 0 max_overlap_ratio = 0 max_jaccard = 0 d_GT_volumes = d_d_gt_res_overlap.setdefault( "GT_volumes", {} ) #d_d_gt_res_overlap : key = gt-label, value = dict (key = RES label, value = nb of overlapping pixels with gt-label) nb_pixel_self = d_GT_volumes.setdefault(gt_label, 0) d_GT_volumes[gt_label] = nb_pixel_self + np.sum(a_bool_gt) d_res_overlap = d_d_gt_res_overlap.setdefault(gt_label, {}) for res_label in a_res_labels_overlap: nb_res_label_pix = np.sum(a_res == res_label) nb_overlap_pix = a_bincount_overlap[res_label] jaccard = (nb_overlap_pix) / (nb_gt_label_pix + nb_res_label_pix - nb_overlap_pix) overlap_ratio = nb_overlap_pix / nb_gt_label_pix if overlap_ratio > max_overlap_ratio: max_overlap_label = res_label max_overlap_ratio = overlap_ratio max_jaccard = jaccard rec = " GT_label {0} ({4} pixels) overlaps with RES_label {1}({5} pixels) with {6} pixels overlap. \n Jaccard = {3} with ratio (nb pix overlap/nb pix GT) = {2}\n".format( gt_label, res_label, (nb_overlap_pix / nb_gt_label_pix), jaccard, nb_gt_label_pix, nb_res_label_pix, nb_overlap_pix) nb_pixel_overlap = d_res_overlap.setdefault(res_label, 0) d_res_overlap[res_label] += nb_overlap_pix if verbose: print(rec, file=output) SEG_score = 0 if max_overlap_ratio < 0.5: SEG_score = 0 rec = " SEG_score = 0 because max_overlap_ratio < 0.5 !" else: SEG_score = max_jaccard rec = " SEG_score = {0} for gt_label {1} overlapping res_label {2} ".format( SEG_score, gt_label, max_overlap_label) SEG_counter += 1 SEG_sum += SEG_score jaccard_sum += max_jaccard if verbose: print(rec, file=output) #write visual if output_folder: # f_output.write(rec) fh.d_save_info[ 'f_name'] = "GT_{0}(z{5})_vs_RES_{1}_SEG={2}|jac={3}|overlap={4}".format( gt_label, max_overlap_label, round(SEG_score, 2), round(max_jaccard, 2), round(max_overlap_ratio, 2), str(ix_z).zfill(3)) a_output = np.zeros_like(a_gt) # a_output [a_gt == gt_label] = gt_label # a_output [a_res == max_overlap_label] = max_overlap_label # a_output [(a_gt == gt_label) & (a_res == max_overlap_label)] = np.round(SEG_score*100) a_output[a_gt == gt_label] = 1 a_output[a_res == max_overlap_label] = 2 a_output[(a_gt == gt_label) & (a_res == max_overlap_label)] = 3 fh.save_data(a_output, verbose=verbose) # if output_folder:f_output.close() a_res_labels = np.unique(a_res) a_res_labels = np.delete(a_res_labels, [0]) if list(a_gt_labels): # a slice with no GT is skipped (=SEG score rules) for res_label in a_res_labels: d_RES_volumes = d_d_gt_res_overlap.setdefault( "RES_volumes", {} ) #d_d_gt_res_overlap : key = gt-label, value = dict (key = RES label, value = nb of overlapping pixels with gt-label) nb_pixel_self = d_RES_volumes.setdefault(res_label, 0) d_RES_volumes[res_label] = nb_pixel_self + np.sum( a_res == res_label) if res_label == 2: print(np.sum(a_res == res_label), 'for a total of ', d_RES_volumes[res_label]) return SEG_sum, jaccard_sum, SEG_counter, d_d_gt_res_overlap
def getFileByName(filename): fileHandler = FileHandler(5) path = fileHandler.getFilePathByDiskName(filename) return static_file(filename,root=path)