def sync_run(self):
storage_conf = self.app.config.get_section_dict('storage')
archive_conf = self.app.config.get_section_dict('archive')
swestore_paths = set(
self.app.config.get_section_dict('archive').get(
'swestore_staging').split(','))
run = self.pargs.tarball if self.pargs.tarball else self.pargs.flowcell
swestore_dir = get_path_swestore_staging(run, swestore_dir)
servers = [server for server in storage_conf.keys()]
server = platform.node().split('.')[0].lower()
flowcell = self.pargs.flowcell
if not flowcell:
self.app.log.error("Flowcell parameter is required.")
return
if server in servers:
#Find the run of the run directories
run_dir = ''
for d in [d.lstrip() for d in storage_conf.get(server).split(',')]:
if os.path.exists(os.path.join(d, flowcell)):
run_dir = os.path.join(d, flowcell)
if not run_dir:
self.app.log.error(
"Run {} not found on the server.".format(flowcell))
return
cl = [
'rsync', '--recursive', run_dir,
'{}@{}:{}'.format(archive_conf.get('user'),
archive_conf.get('server'), swestore_dir)
]
try:
subprocess.check_call(cl)
except subprocess.CalledProcessError():
self.app.log.error("Rsync of run {} failed, please check that you have " \
"the correct username and server in your pm.conf file, " \
"current ones are {} and {}".format(flowcell, archive_conf.get('user'),
archive_conf.get('server')))
return
else:
self.app.log.warn("You're running the cleanup functionality in {}. But this " \
"server doen't seem to be on your pm.conf file. Are you on the correct server?".format(server))
def sync_run(self):
storage_conf = self.app.config.get_section_dict('storage')
archive_conf = self.app.config.get_section_dict('archive')
swestore_paths = set(self.app.config.get_section_dict('archive').get('swestore_staging').split(','))
run = self.pargs.tarball if self.pargs.tarball else self.pargs.flowcell
swestore_dir = get_path_swestore_staging(run, swestore_dir)
servers = [server for server in storage_conf.keys()]
server = platform.node().split('.')[0].lower()
flowcell = self.pargs.flowcell
if not flowcell:
self.app.log.error("Flowcell parameter is required.")
return
if server in servers:
#Find the run of the run directories
run_dir = ''
for d in [d.lstrip() for d in storage_conf.get(server).split(',')]:
if os.path.exists(os.path.join(d, flowcell)):
run_dir = os.path.join(d, flowcell)
if not run_dir:
self.app.log.error("Run {} not found on the server.".format(flowcell))
return
cl = ['rsync',
'--recursive',
run_dir,
'{}@{}:{}'.format(archive_conf.get('user'),
archive_conf.get('server'),
swestore_dir)]
try:
subprocess.check_call(cl)
except subprocess.CalledProcessError():
self.app.log.error("Rsync of run {} failed, please check that you have " \
"the correct username and server in your pm.conf file, " \
"current ones are {} and {}".format(flowcell, archive_conf.get('user'),
archive_conf.get('server')))
return
else:
self.app.log.warn("You're running the cleanup functionality in {}. But this " \
"server doen't seem to be on your pm.conf file. Are you on the correct server?".format(server))
def swestore(self):
"""This function is the entry point for tasks having to do with packaging and sending runs to swestore
"""
db_info = self.app.config.get_section_dict('db')
f_conn = FlowcellRunMetricsConnection(username=db_info.get('user'),
password=db_info.get('password'),
url=db_info.get('url'))
swestore_paths = set(self.config.get('archive','swestore_staging').split(','))
run = self.pargs.tarball if self.pargs.tarball else self.pargs.flowcell
swestore_dir = get_path_swestore_staging(run, swestore_paths)
# Create a tarball out of the run folder
if self.pargs.package_run:
# We require a flowcell argument
if not self._check_pargs(["flowcell"]):
return
self.pargs.tarball = package_run(self, swestore_dir, **vars(self.pargs))
if not self.pargs.tarball:
self.log.error("No tarball was created, exiting")
return
if self.pargs.clean:
rm_run(self,self.config.get('archive','root'), flowcell=self.pargs.flowcell)
if self.pargs.clean_from_staging:
#Check that the run has been archived on the NAS before removing it, otherwise it will keep synching
if self.pargs.flowcell in f_conn.get_storage_status('NAS_nosync').keys():
rm_run(self, swestore_dir, flowcell=self.pargs.flowcell)
else:
self.log.warn("Run storage status is not NAS_nosync, not removing run from swestore_stage!")
if not self.pargs.tarball:
self.log.error("Required argument --tarball was not specified")
return
if not os.path.exists(os.path.join(swestore_dir, self.pargs.tarball)):
self.log.error("Tarball {} does not exist".format(self.pargs.tarball))
return
# Upload a tarball to a remote host
if self.pargs.remote_upload:
result = upload_tarball(self,
**dict(self.config.get_section_dict('archive').items() + vars(self.pargs).items()))
if not result:
return
if self.pargs.clean:
rm_tarball(self,tarball=self.pargs.tarball)
# Send the tarball to Swestore using irods
if self.pargs.send_to_swestore:
result = send_to_swestore(self,**dict(self.config.get_section_dict('archive').items() + vars(self.pargs).items()))
if not result:
# If archiving failed, we need to give a non-zero exit code in order for a remote instance to detect the failure
sys.exit(1)
if self.pargs.clean:
rm_tarball(self,tarball=self.pargs.tarball)
#Set the run as archived in StatusDB
fc_id = self.pargs.flowcell if self.pargs.flowcell else self.pargs.tarball.split('.')[0]
fc_db_id = f_conn.id_view.get(fc_id)
if fc_db_id:
f_conn.set_storage_status(fc_db_id, 'swestore_archived')
else:
self.log.warn("Flowcell {} not found in the database, not changing status.".format(fc_id))
# Log to statusdb
if self.pargs.log_to_db:
# implement this
raise NotImplementedError("logging to db functionality not implemented")
