def upload_file(self):
plpt = os.path.join(self.network['host'],self.contest_id)
for problem in self.problems:
if os.path.exists(os.path.join(plpt, problem, 'tests')):
shutil.rmtree(os.path.join(plpt, problem, 'tests'))
#copy_with_leading_zeroes = lambda src,dst : shutil.copy2(src,
# os.path.join(
# os.path.split(src)[0],
# os.path.split(src)[1].split('.')[0].zfill(2),
# os.path.split(src)[1].split('.')[1]
# if len(os.path.split(src)[1].split('.')) > 1 else None))
shutil.copytree(src = os.path.join(problem,'.tests'), dst = os.path.join(plpt,problem, 'tests'))
for filename in os.listdir(os.path.join(plpt,problem, 'tests')):
if '.a' in filename:
os.rename(os.path.join(os.path.join(plpt,problem, 'tests', filename)),
os.path.join(os.path.join(plpt,problem, 'tests', filename.zfill(4))))
else:
os.rename(os.path.join(os.path.join(plpt,problem, 'tests', filename)),
os.path.join(os.path.join(plpt,problem, 'tests', filename.zfill(2))))
conf = PackageConfig.get_config(problem)
#self.archiver.add(conf['checker'], os.path.join(problem, os.path.split(conf['checker'])[-1]))
checker_name = os.path.split(conf['checker'])[1]
if os.path.exists(os.path.join(plpt, problem, checker_name)):
shutil.unlink(os.path.join(plpt, problem, checker_name))
shutil.copy2(src = os.path.join(problem, checker_name), dst = os.path.join(plpt,problem))
def create_db(self, accessions, emailAddress=None, JVMmemory=None):
sortedAccessionString = ", ".join([phylo.genbank.parse_accession_str(acc) for acc in sorted(accessions)])
databaseId = hashlib.sha256(sortedAccessionString.encode('utf-8')).hexdigest()[:55]
# if the database is not installed, we need to make it
if not self.has_genome(databaseId):
config_file = os.path.join(os.path.dirname(os.path.realpath(self.install_and_get_path())), 'snpEff.config')
data_dir = get_data_dir(config_file)
# if the data directory specified in the config is absolute, use it
# otherwise get the data directory relative to the location of the config file
if os.path.isabs(data_dir):
outputDir = os.path.join(data_dir, databaseId)
else:
outputDir = os.path.realpath(os.path.join(os.path.dirname(config_file), data_dir, databaseId))
phylo.genbank.fetch_full_records_from_genbank(
sorted(accessions),
outputDir,
emailAddress,
forceOverwrite=True,
combinedFilePrefix="genes",
removeSeparateFiles=False
)
# create a temp config file in the same location as the original
# since the data dir for the built database has a relative
# location by default
tmp_config_file = config_file+".temp"
# build a new snpEff database using the downloaded genbank file
# Note that if the snpEff command fails for some reason,
# we should take care to not include an entry for the database
# in the config file, which is the record of installed databases.
# In the event the build fails, the config file must reflect
# databases present in the data_dir, otherwise calls to execute snpEff will
# fail if the data_dir lacks databases for entries listed in the config file.
# make a temp copy of the old config file
shutil.copyfile(config_file, tmp_config_file)
# add the new genome to the temp config file
# since snpEff will not attempt to build unless the
# database is in the config file
add_genomes_to_snpeff_config_file(tmp_config_file, [(databaseId, sortedAccessionString, sortedAccessionString)])
try:
args = ['-genbank', '-v', databaseId, "-c", tmp_config_file]
self.execute('build', args, JVMmemory=JVMmemory)
except:
# remove temp config file if the database build failed
shutil.unlink(tmp_config_file)
raise
# copy the temp config including the built database
# if the execute('build') command did not raise an exception
shutil.move(tmp_config_file, config_file)
self.known_dbs.add(databaseId)
self.installed_dbs.add(databaseId)
def create_db(self, accessions, emailAddress=None, JVMmemory=None):
sortedAccessionString = ", ".join([util.genbank.parse_accession_str(acc) for acc in sorted(accessions)])
databaseId = hashlib.sha256(sortedAccessionString.encode('utf-8')).hexdigest()[:55]
# if the database is not installed, we need to make it
if not self.has_genome(databaseId):
config_file = os.path.join(os.path.dirname(os.path.realpath(self.install_and_get_path())), 'snpEff.config')
data_dir = get_data_dir(config_file)
# if the data directory specified in the config is absolute, use it
# otherwise get the data directory relative to the location of the config file
if os.path.isabs(data_dir):
outputDir = os.path.join(data_dir, databaseId)
else:
outputDir = os.path.realpath(os.path.join(os.path.dirname(config_file), data_dir, databaseId))
util.genbank.fetch_full_records_from_genbank(
sorted(accessions),
outputDir,
emailAddress,
forceOverwrite=True,
combinedFilePrefix="genes",
removeSeparateFiles=False
)
# create a temp config file in the same location as the original
# since the data dir for the built database has a relative
# location by default
tmp_config_file = config_file+".temp"
# build a new snpEff database using the downloaded genbank file
# Note that if the snpEff command fails for some reason,
# we should take care to not include an entry for the database
# in the config file, which is the record of installed databases.
# In the event the build fails, the config file must reflect
# databases present in the data_dir, otherwise calls to execute snpEff will
# fail if the data_dir lacks databases for entries listed in the config file.
# make a temp copy of the old config file
shutil.copyfile(config_file, tmp_config_file)
# add the new genome to the temp config file
# since snpEff will not attempt to build unless the
# database is in the config file
add_genomes_to_snpeff_config_file(tmp_config_file, [(databaseId, sortedAccessionString, sortedAccessionString)])
try:
args = ['-genbank', '-v', databaseId, "-c", tmp_config_file]
self.execute('build', args, JVMmemory=JVMmemory)
except:
# remove temp config file if the database build failed
shutil.unlink(tmp_config_file)
raise
# copy the temp config including the built database
# if the execute('build') command did not raise an exception
shutil.move(tmp_config_file, config_file)
self.known_dbs.add(databaseId)
self.installed_dbs.add(databaseId)
def addCardToReference(img_src_path,img_dest_path,card_name,reference_table_path):
src = pathlib.Path(img_src_path).resolve()
dst = pathlib.Path(img_dst_path).resolve()
table = pathlib.Path(reference_table_path).resolve()
img_filename = "_" + src.name
dst = dst / img_filename
shutil.copy(src,dst)
shutil.unlink(src)
table_entry = "\n{};{}".format(img_filename,card_name)
with open(table,'a') as t:
t.write(table_entry)
def website_export(self):
main_step=1.0/self.max_depth
progress=0
if not self.progress_callback(progress, _("Starting export")):
return
view_url={}
ctx=self.controller.build_context()
# Pre-seed url translations for base views
for v in self.views:
link="/".join( (ctx.globals['options']['package_url'], 'view', v.id) )
if hasattr(v, 'content'):
if v.content.mimetype == 'text/plain':
ext='.txt'
else:
ext=mimetypes.guess_extension(v.content.mimetype)
if ext is not None:
self.url_translation[link]="%s%s" % (v.id, ext)
else:
self.url_translation[link]=v.id
else:
self.url_translation[link]=v.id
view_url[v]=link
progress=.01
depth=1
links_to_be_processed=list(view_url.values())
while depth <= self.max_depth:
max_depth_exceeded = (depth == self.max_depth)
step=main_step / (len(links_to_be_processed) or 1)
if not self.progress_callback(progress, _("Depth %d") % depth):
return
links=set()
for url in links_to_be_processed:
if not self.progress_callback(progress, _("Depth %(depth)d: processing %(url)s") % locals()):
return
progress += step
content=self.get_contents(url)
(new_links,
used_snapshots,
used_overlays,
used_resources)=self.translate_links(content,
url,
max_depth_exceeded)
links.update(new_links)
# Write contents
self.write_data(url, self.fix_links(content),
used_snapshots,
used_overlays,
used_resources)
links_to_be_processed=links
depth += 1
if not self.progress_callback(0.95, _("Finalizing")):
return
# Copy static video player resources
for (path, dest) in self.video_player.needed_resources():
dest=os.path.join(self.destination, dest)
if os.path.isdir(path):
# Copy tree
if os.path.exists(dest):
# First remove old version
if os.path.isdir(dest):
shutil.rmtree(dest, True)
else:
shutil.unlink(dest)
shutil.copytree(path, dest)
else:
# Copy file
d=os.path.dirname(dest)
if not os.path.isdir(d):
helper.recursive_mkdir(d)
shutil.copy(path, dest)
# Generate video helper files if necessary
self.video_player.finalize()
# Generate a default index.html
name="index.html"
if name in list(self.url_translation.values()):
name="_index.html"
f=open(os.path.join(self.destination, name), 'w', encoding='utf-8')
defaultview=self.controller.package.getMetaData(config.data.namespace, 'default_utbv')
v=self.controller.package.views.get_by_id(defaultview)
if defaultview and v:
default_href=self.url_translation[view_url[v]]
default=_("""<p><strong>You should probably begin at <a href="%(href)s">%(title)s</a>.</strong></p>""") % { 'href': default_href, 'title': self.controller.get_title(v) }
else:
default_href=''
default=''
f.write("""<html><head>%(title)s</head>
<body>
<h1>%(title)s views</h1>
%(default)s
<ul>
%(data)s
</ul></body></html>""" % { 'title': self.controller.package.title,
'default': default,
'data': "\n".join( '<li><a href="%s">%s</a>' % (self.url_translation[view_url[v]],
v.title)
for v in self.views ) })
f.close()
frame="frame.html"
if frame in list(self.url_translation.values()):
frame="_frame.html"
f=open(os.path.join(self.destination, frame), 'w', encoding='utf-8')
f.write("""<html>
<head><title>%(title)s</title></head>
<frameset cols="70%%,30%%">
<frame name="main" src="%(index)s" />
<frame name="video_player" src="" />
</frameset>
</html>
""" % {
'title': self.controller.get_title(self.controller.package),
'index': default_href or name,
})
f.close()
f=open(os.path.join(self.destination, "unconverted.html"), 'w', encoding='utf-8')
f.write("""<html><head>%(title)s - not converted</head>
<body>
<h1>%(title)s - not converted resource</h1>
<p>Advene was unable to export this resource.</p>
</body></html>""" % { 'title': self.controller.get_title(self.controller.package) })
f.close()
self.progress_callback(1.0, _("Export complete"))
def website_export(self):
main_step=1.0/self.max_depth
progress=0
if not self.progress_callback(progress, _("Starting export")):
return
view_url={}
ctx=self.controller.build_context()
# Pre-seed url translations for base views
for v in self.views:
link="/".join( (ctx.globals['options']['package_url'], 'view', v.id) )
if hasattr(v, 'content'):
if v.content.mimetype == 'text/plain':
ext='.txt'
else:
ext=mimetypes.guess_extension(v.content.mimetype)
if ext is not None:
self.url_translation[link]="%s%s" % (v.id, ext)
else:
self.url_translation[link]=v.id
else:
self.url_translation[link]=v.id
view_url[v]=link
progress=.01
depth=1
links_to_be_processed=view_url.values()
while depth <= self.max_depth:
max_depth_exceeded = (depth == self.max_depth)
step=main_step / (len(links_to_be_processed) or 1)
if not self.progress_callback(progress, _("Depth %d") % depth):
return
links=set()
for url in links_to_be_processed:
if not self.progress_callback(progress, _("Depth %(depth)d: processing %(url)s") % locals()):
return
progress += step
content=self.get_contents(url)
(new_links,
used_snapshots,
used_overlays,
used_resources)=self.translate_links(content,
url,
max_depth_exceeded)
links.update(new_links)
# Write contents
self.write_data(url, self.fix_links(content),
used_snapshots,
used_overlays,
used_resources)
links_to_be_processed=links
depth += 1
if not self.progress_callback(0.95, _("Finalizing")):
return
# Copy static video player resources
for (path, dest) in self.video_player.needed_resources():
dest=os.path.join(self.destination, dest)
if os.path.isdir(path):
# Copy tree
if os.path.exists(dest):
# First remove old version
if os.path.isdir(dest):
shutil.rmtree(dest, True)
else:
shutil.unlink(dest)
shutil.copytree(path, dest)
else:
# Copy file
d=os.path.dirname(dest)
if not os.path.isdir(d):
helper.recursive_mkdir(d)
shutil.copy(path, dest)
# Generate video helper files if necessary
self.video_player.finalize()
# Generate a default index.html
name="index.html"
if name in self.url_translation.values():
name="_index.html"
f=open(os.path.join(self.destination, name), 'w')
defaultview=self.controller.package.getMetaData(config.data.namespace, 'default_utbv')
v=self.controller.package.views.get_by_id(defaultview)
if defaultview and v:
default_href=self.url_translation[view_url[v]]
default=_("""<p><strong>You should probably begin at <a href="%(href)s">%(title)s</a>.</strong></p>""") % { 'href': default_href, 'title': self.controller.get_title(v) }
else:
default_href=''
default=''
f.write("""<html><head>%(title)s</head>
<body>
<h1>%(title)s views</h1>
%(default)s
<ul>
%(data)s
</ul></body></html>""" % { 'title': self.controller.package.title,
'default': default,
'data': "\n".join( '<li><a href="%s">%s</a>' % (self.url_translation[view_url[v]],
v.title)
for v in self.views ) })
f.close()
frame="frame.html"
if frame in self.url_translation.values():
frame="_frame.html"
f=open(os.path.join(self.destination, frame), 'w')
f.write("""<html>
<head><title>%(title)s</title></head>
<frameset cols="70%%,30%%">
<frame name="main" src="%(index)s" />
<frame name="video_player" src="" />
</frameset>
</html>
""" % {
'title': self.controller.get_title(self.controller.package),
'index': default_href or name,
})
f.close()
f=open(os.path.join(self.destination, "unconverted.html"), 'w')
f.write("""<html><head>%(title)s - not converted</head>
<body>
<h1>%(title)s - not converted resource</h1>
<p>Advene was unable to export this resource.</p>
</body></html>""" % { 'title': self.controller.get_title(self.controller.package) })
f.close()
self.progress_callback(1.0, _("Export complete"))
nclusters,
)
instruction_filename = os.path.abspath("fuzzy_k/fuzzykinstructions-%s-%s" % (str(min_rpkm), argv[8]))
inst_file = open(instruction_filename, "w")
inst_file.write(my_str)
inst_file.close()
# run aerie in a temporary directory
tmpdir = tempfile.mkdtemp()
# move cdt_file locally (is this really necessary)
shutil.copy(argv[2], os.path.join(tmpdir, os.path.basename(argv[2])))
# Grant used to copy the excecutable into the local directory, too
try:
retcode = subprocess.call("arie < %(instruction_filename)s" % locals(), shell=True, cwd=tmpdir)
if retcode < 0:
print >>sys.stderr, "Child was terminated by signal", -retcode
else:
print >>sys.stderr, "Child returned", retcode
except OSError, e:
print >>sys.stderr, "Execution failed:", e
shutil.move(os.path.join(tmpdir, "alls.fct"), centroid_filename)
shutil.move(os.path.join(tmpdir, "alls.mb"), membership_filename)
shutil.unlink(tmpdir)
instruction_filename = os.path.abspath('fuzzy_k/fuzzykinstructions-%s-%s' %
(str(min_rpkm), argv[8]))
inst_file = open(instruction_filename, 'w')
inst_file.write(my_str)
inst_file.close()
# run aerie in a temporary directory
tmpdir = tempfile.mkdtemp()
# move cdt_file locally (is this really necessary)
shutil.copy(argv[2], os.path.join(tmpdir, os.path.basename(argv[2])))
# Grant used to copy the excecutable into the local directory, too
try:
retcode = subprocess.call("arie < %(instruction_filename)s" % locals(),
shell=True,
cwd=tmpdir)
if retcode < 0:
print >> sys.stderr, "Child was terminated by signal", -retcode
else:
print >> sys.stderr, "Child returned", retcode
except OSError, e:
print >> sys.stderr, "Execution failed:", e
shutil.move(os.path.join(tmpdir, "alls.fct"), centroid_filename)
shutil.move(os.path.join(tmpdir, "alls.mb"), membership_filename)
shutil.unlink(tmpdir)
