def add_mapping_to_db():
f = FileHandler('../../data/mapping.txt')
for r in f:
tokens = r.split('\t')
rsname, gn, mn, version = tokens[0:4]
mapped_count, unique_count, pos_unique_count, neg_unique_count, s_name = tokens[4:9]
params = tokens[9:]
rs = Raw_sample.objects.get(name=rsname)
genome = Genome.objects.get(assembly_name=gn)
method = Method.objects.get(method_name=mn, method_version=version)
sid = transaction.savepoint()
try:
'''
param_set = Parameter_set(method_obj = method)
param_set.save()
for p in params:
t = p.split(':')
if len(t) == 1:
t.append('')
param = Parameter(param_set = param_set, param_name = t[0], param_value = t[1])
param.save()
'''
param_set = add_params_to_db(params, method)
mapping = Mapping(method_obj=method, genome_obj=genome, params=param_set, sample_obj=rs)
mapping.save()
sample = Sample(name = s_name, mapped_tags = mapped_count, unique_tags = unique_count, positive_unique_count = pos_unique_count, negative_unique_count = neg_unique_count, mapping_obj = mapping)
sample.save()
transaction.savepoint_commit(sid)
except IntegrityError:
print 'error'
transaction.savepoint_rollback(sid)
f.close()
class Main:
def __init__(self):
self.soupHandler = SoupHandler(SystemInfo.initUri)
self.downloader = Downloader()
self.fileHandler = FileHandler()
self.path = SystemInfo.path
self.name = SystemInfo.name
def introduce(self):
print("kocw 다운로드 프로그램.")
print("대학교 자체 cdn 전용입니다.")
print("url 수정은 소스코드를 직접 수정하세요.")
def end(self):
print("다운로드 완료 되었습니다.")
def run(self):
self.introduce()
self.fileHandler.safeMkdir("%s/output" % self.path)
self.fileHandler.safeMkdir("%s/output/%s" % (self.path, self.name))
table = self.soupHandler.findTable()
parsedOnClickStrings = self.soupHandler.parseOnClick(table)
self.downloader.startIteratingDownload(parsedOnClickStrings)
self.end()
def add_peakcalling_to_db():
f = FileHandler("../../data/peakcalling.txt")
for r in f:
tokens = r.split('\t')
s_name, method_name, method_version = tokens[0:3]
peak_file = tokens[3]
params = tokens[4:]
print "a"
sample = Sample.objects.get(name=s_name)
method = Method.objects.get(method_name = method_name, method_version = method_version)
print 'b'
'''
run = models.ForeignKey(Call_peak)
chrom = models.CharField(max_length=40)
start = models.PositiveIntegerField(null=False) #0-based
end = models.PositiveIntegerField(null=False) #1-based
size = models.PositiveIntegerField(null=False)
strand = models.CharField(max_length=1, choices=(('+', 'Positive'), ('-', 'Negative'), ('.', 'Unspecified')))
tag_count = models.FloatField(null=False)
summit_pos_pileup = models.PositiveSmallIntegerField(null=False) #Origin summit location by peak calling method
summit_val_pileup = models.FloatField(null=False)
summit_pos_5 = models.PositiveSmallIntegerField()
summit_val_5 = models.FloatField()
p_value = models.FloatField()
q_value = models.FloatField()
fold = models.FloatField()
'''
sid = transaction.savepoint()
try:
param_set = add_params_to_db(params, method)
callpeak = Call_peak(sample_obj = sample, method_obj = method, params = param_set)
callpeak.save()
sid = transaction.savepoint()
ph = FileHandler(peak_file)
for p in ph:
ptokens = p.split('\t')
chrom, start, end, length, strand, abs_summit, pileup, pileup_5, p_val, fold, q_val, _, tc, rpkm = ptokens
start = int(start)
abs_summit = int(abs_summit)
tc = float(tc)
#print p
newp = Peak(run=callpeak, chrom=chrom, start=start-1, end=end,
size=length, strand=strand, tag_count=tc, summit_pos_pileup=abs_summit-1-start,
summit_val_pileup = pileup, summit_pos_5 = abs_summit-1-start, summit_val_5=pileup_5,
p_value = p_val, q_value = q_val, fold = fold)
newp.save()
transaction.savepoint_commit(sid)
ph.close()
except IntegrityError, msg:
print msg
transaction.savepoint_rollback(sid)
def do_upload():
upload = request.files.get('upload')
url = request.forms.get('url')
content_type = upload.content_type
name, ext = os.path.splitext(upload.filename)
save_path = '/tmp'
upload.filename = str(abs(hash(name)))+ ext
upload.save(save_path,overwrite=True) # appends upload.filename automatically
fileHandler = FileHandler(5)
fileId,physicalName = fileHandler.save('/tmp/'+upload.filename ,url)
return physicalName
class Downloader:
def __init__(self):
self.path = SystemInfo.path
self.name = SystemInfo.name
self.videoUriFormat = SystemInfo.videoUriFormat
self.fileHandler = FileHandler()
self.strParser = StrParser()
def startIteratingDownload(self, parsedOnclicks):
count = 0
for line in parsedOnclicks:
count += 1
uri = self.strParser.findSubStr(line, "'", "'")
print(uri)
if not SystemInfo.pdfIncluded or count % 2 == 1:
code = uri.split("/")[-1]
self.fileHandler.safeMkdir("%s/output/%s/video" %
(self.path, self.name))
downloadUri = self.videoUriFormat % code
print("%s 다운로드 중..." % (downloadUri))
res = requests.get(downloadUri)
if SystemInfo.pdfIncluded:
number = (count + 1) // 2
self.fileHandler.saveBinaryFile(res.content, "video", number,
"mp4")
else:
print("%s 다운로드 중..." % uri)
res = requests.get(uri)
self.fileHandler.safeMkdir("%s/output/%s/pdf" %
(self.path, self.name))
number = (count + 1) // 2
self.fileHandler.saveBinaryFile(res.content, "pdf", number,
"pdf")
def main():
if len(sys.argv) != 1 + 1:
print("Provide the android PhotoScan file folder as argument.")
return
directory = sys.argv[1]
handler = FileHandler()
w = FileWatcher(directory, handler.processNewFile)
w.run()
try:
while True:
seriesName = input("Current series name: ")
seriesDate = input("Series Date (YYYY:MM) : ")
handler.setSeriesName(seriesName)
handler.setSeriesDate(seriesDate)
print("Running series...")
# command loop
command = ""
while command != "n":
command = input("[d]elete last image or [n]ext series? ")
if command == "d":
handler.deleteLastFile()
except KeyboardInterrupt:
print("") # force newline in terminal
w.stop()
sys.exit()
def add_method_to_db():
f = FileHandler("../../data/methods.txt")
sid = transaction.savepoint()
for r in f:
tokens = r.split('\t')
m = Method(method_name=tokens[0],
method_type=tokens[1],
method_version=tokens[2])
sid = transaction.savepoint()
try:
m.save()
transaction.savepoint_commit(sid)
except DatabaseError:
transaction.savepoint_rollback(sid)
f.close()
def add_genome_to_db():
f = FileHandler("../../data/genomes.txt")
#sid = transaction.savepoint()
for r in f:
tokens = r.split('\t')
g = Genome(assembly_name = tokens[0],
alternative_assembly_name = tokens[1],
species = tokens[2],
size = tokens[3])
sid = transaction.savepoint() #create save points before transaction for rollback if error occurs.
try:
g.save()
transaction.savepoint_commit(sid)
except IntegrityError:
print "error"
transaction.savepoint_rollback(sid)
f.close()
def add_chromosomes_to_db():
genomes = Genome.objects.all()
if len(genomes) <= 0:
print "No genomes. Abort."
else:
for g in genomes:
f = FileHandler('../../data/%s.chrom.sizes'%g.assembly_name)
#sid = transaction.savepoint()
for r in f:
cname, csize = r.split()
chrom = Chromosome(name=cname, length=csize, genome_obj=g)
sid = transaction.savepoint()
try:
chrom.save()
transaction.savepoint_commit(sid)
except IntegrityError:
transaction.savepoint_rollback(sid)
print 'error'
f.close()
def make_image_sequence(input_file, output_folder, prefix='man_seg_000_'):
'''
the fiji 'image sequence' can lead to the error :'Incorrect count for "ColorMap" when using Tiffreader, therefore made my own
'''
filehandler = FileHandler()
filehandler.d_load_info['f_name'] = str(input_file)
filehandler.d_save_info['save_dir'] = str(output_folder)
a_img = filehandler.load_tif()
for ix_z, a_slice in enumerate(a_img):
suffix = "{:03d}".format(ix_z)
filehandler.d_save_info['f_name'] = "{0}{1}".format(prefix, suffix)
filehandler.save_data(a_slice)
return
def count(self, most_common_words, *args):
"""
:param dir_path: String - Full path to directory where files reside
:param most_common_words: Integer - Number of most common words in all files
:return: Integer - Number of most common words
"""
_counter = Counter()
self.most_common_words = most_common_words
_root_dir = DirectoryHandler()
for path in map(abspath, args):
if isfile(path):
_counter += FileHandler().file_word_count(filename=path)
else:
with ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor:
future_file_counts = {executor.submit(FileHandler().file_word_count, _filename):
_filename for _filename in _root_dir.scan_dir(path)}
for future in as_completed(future_file_counts):
_counter += future.result()
self.logger.debug("size of counter is {0} MB".format(round(getsizeof(_counter) / 1024 / 1024), 3))
return _counter.most_common(self.most_common_words)
class Chatbot:
fh = FileHandler('newconversa.txt')
def __init__(self, nome):
self.nome = nome
resp = self.intro()
if resp.upper() == 'S':
while self.conversar().lower() != '.sair':
self.conversar()
else:
self.tchau()
def intro(self):
print("{}: Olá meu nome é {}".format(self.nome, self.nome))
print("{}: Eu sou uma I.A. que irá aprender com você".format(
self.nome))
resp = input("{}: Vamos começar? (S/N): ".format(self.nome))
return resp
def conversar(self):
r = input("Você: ")
if r == '.sair':
return r
if self.fh.comunica(r) != 0:
print(self.nome + ": " + self.fh.comunica(r))
else:
self.aprender(r)
return r
def aprender(self, p):
print(self.nome + ": Não sei responder!")
r = input(self.nome + ": Qual seria a resposta para essa pergunta? ")
self.fh.gravarArquivo(p, r)
def tchau(self):
print("{}: Tchau".format(self.nome))
class TestHander(unittest.TestCase):
dbHandler = None;
fileHandler = None;
def setUp(self):
self.dbHandler = DBHandler()
self.fileHandler = FileHandler(5)
def testFileHanderSave(self,url = 'aa'):
path = '/tmp/license.txt'
import os
if not os.path.isfile(path):
os.mknod(path)
fileId,physicalName = self.fileHandler.save(path,url)
return fileId,physicalName
def testGetFilePathByPhysicalName(self):
fileId,physicalName = self.testFileHanderSave()
path = self.fileHandler.getFilePathByDiskName(physicalName)
print 'path :'+ path
def testGetFiletypeByURL(self):
url = 'http://www.abc.com'
self.testFileHanderSave(url)
print 'file type is :' + self.fileHandler.getTypeByUrl(url)
def testGetFileByID(self):
fileId,physicalName = self.testFileHanderSave()
opendFile = self.fileHandler.getFileByID(fileId)
def testGetFiletypeByID(self):
fileId,physicalName = self.testFileHanderSave()
print self.fileHandler.getTypeByID(fileId)
def testGetTotalFileCount(self):
num = self.fileHandler.getTotalFileCount()
print "file count : " + str(num)
def testExecute(self):
sql = 'select * from files'
rows = self.dbHandler.excute(sql)
from watchdog.observers import Observer
import os
import json
import time
from fileHandler import FileHandler
tracked_folder = input('Type the path to the folder to be tracked: \n')
destination_folder = input('Type the path to the destination folder: \n')
event_handler = FileHandler(tracked_folder, destination_folder)
observer = Observer()
observer.schedule(event_handler, tracked_folder, recursive=True)
observer.start()
try:
while True:
time.sleep(10)
except KeyboardInterrupt:
observer.stop()
observer.join()
def setUp(self):
self.dbHandler = DBHandler()
self.fileHandler = FileHandler(5)
def __init__(self):
self.path = SystemInfo.path
self.name = SystemInfo.name
self.videoUriFormat = SystemInfo.videoUriFormat
self.fileHandler = FileHandler()
self.strParser = StrParser()
if __name__ == "__main__":
'''
This script receive a name of an actor or actress and print the information of all the films where
he/she has appear.
Only full names are supported as they appear in the IMDB database, but if the name is not found, a
broad spectrum search is performed. In this search, the name is splitted in all the parts it have
and print the info of all the actors that fit to any of those partial names.
'''
option = ""
while(option != "0"):
print(("Please write the name of an actor or actress to know all the films where he/she appear "
+ "(Only full names supported)"
+ " , write 0 to exit:"))
option = str(input('Enter the actor name option: '))
db = FileHandler()
writer = Writer()
db.openActDB()
if option != "0" and option != "":
films = db.searchActor(option)
if len(films) > 0:
writer.printFilmsDataFromActor(option, films)
else:
print(("%s not found") % (option))
print(("A broad spectrum search will take place, wait a moment"))
result = db.broadSpectrumSearchActor(option)
print(("All the actors whose name contains any of the words provided will be listed now"))
for key in result.keys():
writer.printFilmsDataFromActor(key, result[key])
def initialize_filehandler():
filehandler = FileHandler()
filehandler.d_save_info['save_dir'] = OUTPUT_FOLDER_ROOT
#print("DEBUG1: filehandler {0}, OUTPUT_FOLDER_ROOT {1}, filehandler.get_save_location() {2}".format(filehandler.__repr__(), OUTPUT_FOLDER_ROOT,filehandler.get_save_location()))
if ic_timestamp_subfolder:
if IMG_RAW_FILE:
filehandler.d_save_info['sub_dir_1'] = str(
datetime.datetime.now()).replace(":", "_").replace(
" ", "_") + "_" + IMG_RAW_FILE[0:-4]
else:
filehandler.d_save_info['sub_dir_1'] = str(
datetime.datetime.now()).replace(":",
"_").replace(" ", "_")
filehandler.set_save_info_root()
#print("DEBUG2: filehandler {0}, OUTPUT_FOLDER_ROOT {1}, filehandler.get_save_location() {2}".format(filehandler.__repr__(), OUTPUT_FOLDER_ROOT,filehandler.get_save_location()))
filehandler.d_load_info['load_dir'] = IMG_RAW_DIR
filehandler.take_snapshot()
#print("DEBUG3: filehandler {0}, OUTPUT_FOLDER_ROOT {1}, filehandler.get_save_location() {2}".format(filehandler.__repr__(), OUTPUT_FOLDER_ROOT,filehandler.get_save_location()))
print('SDT_MAIN.py:all info will be written to : ',
filehandler.get_save_location())
return filehandler
def __init__(self):
self.soupHandler = SoupHandler(SystemInfo.initUri)
self.downloader = Downloader()
self.fileHandler = FileHandler()
self.path = SystemInfo.path
self.name = SystemInfo.name
'''
dw = DownloadFromWeb("ftp.fu-berlin.de","pub/misc/movies/database" )
download = False
option = 0
try:
option = int(input("Do you want to download the new version of the databases if available? (1=yes, 0= no)"))
if option != 0 and option != 1:
raise ValueError()
except ValueError:
print("Option not found, assuming default value: no")
if option == 1:
download = True
result = dw.fetch("actresses.list.gz", download)
print(result)
result = dw.fetch("actors.list.gz", download)
print(result)
print("All fetched")
print("Loading actors")
actorHandler = FileHandler()
actorHandler.open("actors.list.gz")
actorHandler.loadData()
print("Actors loaded, closing files")
actorHandler.close()
print("Loading actresses")
actressHandler = FileHandler()
actressHandler.open("actresses.list.gz")
actressHandler.loadData()
print("Actresses loaded, closing files")
actressHandler.close()
#CHECKS
Stack.check_memory_allocation(verbose=prm['verbose'])
#<<<end of stack loop
finish_last_stack(Stack.curr_stack)
if prm['DBG']:
append_summary_of_4Dcell_txt_repr()
append_labelling_chronology_to_txt_log()
feat_tracker.write_excel_summary_4D_v2(fh)
# Stack.save_4D_RGBA_clusterview(fh,nb_channels=1,extra_channel=get_extra_channel())
Stack.save_4D_RGBA_clusterview(fh,nb_channels=1)
Stack.agg_temp_viz_files(fh,search_string='raw_clusterview')
Stack.agg_temp_viz_files(fh,search_string='labelview')
#Stack.save_4D_clusterview(fh)
Stack.init_class_variables()
if prm['DBG']:print('{0} datamodel inconsistencies found across all stacks'.format(nb_inconsistencies))
remove_temp_files()
toc = time.time()
if prm['verbose']:print('runtime_spheresDT = ', toc-tic)
return
if __name__ == '__main__':
param_xml = read_parms()
fh = FileHandler.init_fh(param_xml.prm)
spheres_DT(param_xml,fh)
if __name__ == "__main__":
'''
This script receive 2 actors o actresses names and the maximum distance between them and
return the minimum number of relation (tipe "work with") between them.
Remember that the higher distance, the longer the algorithm will last (default value is 11
and the minimum is 3).
'''
option = ""
while(option != "0"):
print(("Please write the name of two actors/actresses (Only full names supported)"
+ " to see if they are related , write 0 to exit:"))
option = str(input('Enter the actor/actress name: '))
if option != "" or option != "0":
option2 = str(input('Enter the actor/actress name: '))
if option2 != "" or option2 != "0":
db = FileHandler()
db.openActDB()
db.openFilmsDB()
try:
option3 = int(input('Enter maximum length of path to follow, the higher number, the longer it takes (minimum 3, default 11):'))
except:
option3 = 11
if option3 < 3:
option3 = 3
shortestPath = ShortestPath(db, limit=option3)
result = shortestPath.findPath(origin=option, destiny=option2)
if result is None:
print("One of the actor/actress names doesn't exists.")
elif type(result) is int and result == -1:
print("No relation found within " + option3 + " steps")
elif type(result) is Node:
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with coprotagositas. If not, see <http://www.gnu.org/licenses/>.
'''
from fileHandler import FileHandler
from writer import Writer
if __name__ == "__main__":
''' This script recieve 2 actors or actresses and print the films where they are coprotagonist '''
option = ""
while(option != "0"):
print(("Please write the name of two actors/actresses to find the films they made together "
+ "(Only full names supported)"
+ " , write 0 to exit:"))
option1 = str(input('Enter the actor/actress name: '))
if option1 != "" and option1 != "0":
option2 = str(input('Enter the actor/actress name: '))
if option2 != "" and option2 != "0":
db = FileHandler()
writer = Writer()
db.openActDB()
resultsdata = db.searchFilms2actors(option1, option2)
writer.printFilms2Actors(option1, option2, resultsdata)
db.closeActDB()
else:
print("Exiting")
else:
print("Exiting")
def calculate_CTC_SEG_slice_detail(a_gt,
a_res,
output_folder,
ix_z="",
verbose=False,
output=sys.stdout,
d_d_gt_res_overlap={}):
'''
calculate SEG score for 2 slices, with subcalculations
'''
if output_folder:
# f_name = output_folder / "CTC_SEG_detail_calculations.txt"
# f_output = open( f_name ,"w" )
fh = FileHandler()
fh.d_save_info['save_dir'] = str(output_folder)
a_gt_labels = np.unique(a_gt)
a_gt_labels = np.delete(a_gt_labels, [0])
if verbose: print('ground truth labels are ', a_gt_labels, file=output)
SEG_counter = 0
SEG_sum = 0
jaccard_sum = 0
for gt_label in a_gt_labels:
a_bool_gt = a_gt == gt_label
nb_gt_label_pix = np.sum(a_bool_gt)
a_res_overlap = np.where(a_bool_gt, a_res, 0)
a_bincount_overlap = np.bincount(a_res_overlap.flatten())
a_bincount_overlap[0] = 0
a_res_labels_overlap = np.unique(a_res_overlap)
a_res_labels_overlap = np.delete(a_res_labels_overlap, [0])
if verbose:
print(' ground truth label {0} overlaps with res labels {1} '.
format(gt_label, a_res_labels_overlap),
file=output)
max_overlap_label = 0
max_overlap_ratio = 0
max_jaccard = 0
d_GT_volumes = d_d_gt_res_overlap.setdefault(
"GT_volumes", {}
) #d_d_gt_res_overlap : key = gt-label, value = dict (key = RES label, value = nb of overlapping pixels with gt-label)
nb_pixel_self = d_GT_volumes.setdefault(gt_label, 0)
d_GT_volumes[gt_label] = nb_pixel_self + np.sum(a_bool_gt)
d_res_overlap = d_d_gt_res_overlap.setdefault(gt_label, {})
for res_label in a_res_labels_overlap:
nb_res_label_pix = np.sum(a_res == res_label)
nb_overlap_pix = a_bincount_overlap[res_label]
jaccard = (nb_overlap_pix) / (nb_gt_label_pix + nb_res_label_pix -
nb_overlap_pix)
overlap_ratio = nb_overlap_pix / nb_gt_label_pix
if overlap_ratio > max_overlap_ratio:
max_overlap_label = res_label
max_overlap_ratio = overlap_ratio
max_jaccard = jaccard
rec = " GT_label {0} ({4} pixels) overlaps with RES_label {1}({5} pixels) with {6} pixels overlap. \n Jaccard = {3} with ratio (nb pix overlap/nb pix GT) = {2}\n".format(
gt_label, res_label, (nb_overlap_pix / nb_gt_label_pix),
jaccard, nb_gt_label_pix, nb_res_label_pix, nb_overlap_pix)
nb_pixel_overlap = d_res_overlap.setdefault(res_label, 0)
d_res_overlap[res_label] += nb_overlap_pix
if verbose: print(rec, file=output)
SEG_score = 0
if max_overlap_ratio < 0.5:
SEG_score = 0
rec = " SEG_score = 0 because max_overlap_ratio < 0.5 !"
else:
SEG_score = max_jaccard
rec = " SEG_score = {0} for gt_label {1} overlapping res_label {2} ".format(
SEG_score, gt_label, max_overlap_label)
SEG_counter += 1
SEG_sum += SEG_score
jaccard_sum += max_jaccard
if verbose: print(rec, file=output)
#write visual
if output_folder:
# f_output.write(rec)
fh.d_save_info[
'f_name'] = "GT_{0}(z{5})_vs_RES_{1}_SEG={2}|jac={3}|overlap={4}".format(
gt_label, max_overlap_label, round(SEG_score, 2),
round(max_jaccard, 2), round(max_overlap_ratio, 2),
str(ix_z).zfill(3))
a_output = np.zeros_like(a_gt)
# a_output [a_gt == gt_label] = gt_label
# a_output [a_res == max_overlap_label] = max_overlap_label
# a_output [(a_gt == gt_label) & (a_res == max_overlap_label)] = np.round(SEG_score*100)
a_output[a_gt == gt_label] = 1
a_output[a_res == max_overlap_label] = 2
a_output[(a_gt == gt_label) & (a_res == max_overlap_label)] = 3
fh.save_data(a_output, verbose=verbose)
# if output_folder:f_output.close()
a_res_labels = np.unique(a_res)
a_res_labels = np.delete(a_res_labels, [0])
if list(a_gt_labels): # a slice with no GT is skipped (=SEG score rules)
for res_label in a_res_labels:
d_RES_volumes = d_d_gt_res_overlap.setdefault(
"RES_volumes", {}
) #d_d_gt_res_overlap : key = gt-label, value = dict (key = RES label, value = nb of overlapping pixels with gt-label)
nb_pixel_self = d_RES_volumes.setdefault(res_label, 0)
d_RES_volumes[res_label] = nb_pixel_self + np.sum(
a_res == res_label)
if res_label == 2:
print(np.sum(a_res == res_label), 'for a total of ',
d_RES_volumes[res_label])
return SEG_sum, jaccard_sum, SEG_counter, d_d_gt_res_overlap
def getFileByName(filename):
fileHandler = FileHandler(5)
path = fileHandler.getFilePathByDiskName(filename)
return static_file(filename,root=path)
