def sync_run(self): storage_conf = self.app.config.get_section_dict('storage') archive_conf = self.app.config.get_section_dict('archive') swestore_paths = set( self.app.config.get_section_dict('archive').get( 'swestore_staging').split(',')) run = self.pargs.tarball if self.pargs.tarball else self.pargs.flowcell swestore_dir = get_path_swestore_staging(run, swestore_dir) servers = [server for server in storage_conf.keys()] server = platform.node().split('.')[0].lower() flowcell = self.pargs.flowcell if not flowcell: self.app.log.error("Flowcell parameter is required.") return if server in servers: #Find the run of the run directories run_dir = '' for d in [d.lstrip() for d in storage_conf.get(server).split(',')]: if os.path.exists(os.path.join(d, flowcell)): run_dir = os.path.join(d, flowcell) if not run_dir: self.app.log.error( "Run {} not found on the server.".format(flowcell)) return cl = [ 'rsync', '--recursive', run_dir, '{}@{}:{}'.format(archive_conf.get('user'), archive_conf.get('server'), swestore_dir) ] try: subprocess.check_call(cl) except subprocess.CalledProcessError(): self.app.log.error("Rsync of run {} failed, please check that you have " \ "the correct username and server in your pm.conf file, " \ "current ones are {} and {}".format(flowcell, archive_conf.get('user'), archive_conf.get('server'))) return else: self.app.log.warn("You're running the cleanup functionality in {}. But this " \ "server doen't seem to be on your pm.conf file. Are you on the correct server?".format(server))
def sync_run(self): storage_conf = self.app.config.get_section_dict('storage') archive_conf = self.app.config.get_section_dict('archive') swestore_paths = set(self.app.config.get_section_dict('archive').get('swestore_staging').split(',')) run = self.pargs.tarball if self.pargs.tarball else self.pargs.flowcell swestore_dir = get_path_swestore_staging(run, swestore_dir) servers = [server for server in storage_conf.keys()] server = platform.node().split('.')[0].lower() flowcell = self.pargs.flowcell if not flowcell: self.app.log.error("Flowcell parameter is required.") return if server in servers: #Find the run of the run directories run_dir = '' for d in [d.lstrip() for d in storage_conf.get(server).split(',')]: if os.path.exists(os.path.join(d, flowcell)): run_dir = os.path.join(d, flowcell) if not run_dir: self.app.log.error("Run {} not found on the server.".format(flowcell)) return cl = ['rsync', '--recursive', run_dir, '{}@{}:{}'.format(archive_conf.get('user'), archive_conf.get('server'), swestore_dir)] try: subprocess.check_call(cl) except subprocess.CalledProcessError(): self.app.log.error("Rsync of run {} failed, please check that you have " \ "the correct username and server in your pm.conf file, " \ "current ones are {} and {}".format(flowcell, archive_conf.get('user'), archive_conf.get('server'))) return else: self.app.log.warn("You're running the cleanup functionality in {}. But this " \ "server doen't seem to be on your pm.conf file. Are you on the correct server?".format(server))
def swestore(self): """This function is the entry point for tasks having to do with packaging and sending runs to swestore """ db_info = self.app.config.get_section_dict('db') f_conn = FlowcellRunMetricsConnection(username=db_info.get('user'), password=db_info.get('password'), url=db_info.get('url')) swestore_paths = set(self.config.get('archive','swestore_staging').split(',')) run = self.pargs.tarball if self.pargs.tarball else self.pargs.flowcell swestore_dir = get_path_swestore_staging(run, swestore_paths) # Create a tarball out of the run folder if self.pargs.package_run: # We require a flowcell argument if not self._check_pargs(["flowcell"]): return self.pargs.tarball = package_run(self, swestore_dir, **vars(self.pargs)) if not self.pargs.tarball: self.log.error("No tarball was created, exiting") return if self.pargs.clean: rm_run(self,self.config.get('archive','root'), flowcell=self.pargs.flowcell) if self.pargs.clean_from_staging: #Check that the run has been archived on the NAS before removing it, otherwise it will keep synching if self.pargs.flowcell in f_conn.get_storage_status('NAS_nosync').keys(): rm_run(self, swestore_dir, flowcell=self.pargs.flowcell) else: self.log.warn("Run storage status is not NAS_nosync, not removing run from swestore_stage!") if not self.pargs.tarball: self.log.error("Required argument --tarball was not specified") return if not os.path.exists(os.path.join(swestore_dir, self.pargs.tarball)): self.log.error("Tarball {} does not exist".format(self.pargs.tarball)) return # Upload a tarball to a remote host if self.pargs.remote_upload: result = upload_tarball(self, **dict(self.config.get_section_dict('archive').items() + vars(self.pargs).items())) if not result: return if self.pargs.clean: rm_tarball(self,tarball=self.pargs.tarball) # Send the tarball to Swestore using irods if self.pargs.send_to_swestore: result = send_to_swestore(self,**dict(self.config.get_section_dict('archive').items() + vars(self.pargs).items())) if not result: # If archiving failed, we need to give a non-zero exit code in order for a remote instance to detect the failure sys.exit(1) if self.pargs.clean: rm_tarball(self,tarball=self.pargs.tarball) #Set the run as archived in StatusDB fc_id = self.pargs.flowcell if self.pargs.flowcell else self.pargs.tarball.split('.')[0] fc_db_id = f_conn.id_view.get(fc_id) if fc_db_id: f_conn.set_storage_status(fc_db_id, 'swestore_archived') else: self.log.warn("Flowcell {} not found in the database, not changing status.".format(fc_id)) # Log to statusdb if self.pargs.log_to_db: # implement this raise NotImplementedError("logging to db functionality not implemented")