def upload_from_options( parser ): """ Upload files based upon command line options supecified in an OptionParser """ # don't clean tar directory tasks.CLEAN_TAR = False # populate the session so that we are running the same process as the django uploader session = session_data.SessionState() # get rid of redundant file paths filtered = session.files.filter_selected_list(parser.values.file_list) session.files.data_dir = parser.values.work_dir # add a final separator session.files.common_path = os.path.join(parser.values.work_dir, '') # build archive path configuration = instrument_server.InstrumentConfiguration() configuration.instrument_short_name = parser.values.instrument session.proposal_id = parser.values.proposal session.config = configuration session.get_archive_tree() # get the file tuples (local name, archive name) to bundle tuples = session.files.get_bundle_files(parser.values.file_list) tartar = False if parser.values.tartar == 'True': tartar = True tasks.upload_files( bundle_name=parser.values.bundle_name, instrument_name=parser.values.instrument, proposal=parser.values.proposal, file_list=tuples, bundle_size=session.files.bundle_size, groups=parser.groups, server=parser.values.server, user=parser.values.user, password=parser.values.password, tartar=tartar)
def upload_from_options( parser ): """ Upload files based upon command line options supecified in an OptionParser """ # populate the session session = session_data.SessionState() filtered = session.files.filter_selected_list(parser.values.file_list) common_path = os.path.commonprefix(filtered) #get rid of dangling prefixes common_path, tail = os.path.split(common_path) common_path = os.path.join(common_path, '') # used to arc names session.files.common_path = common_path # build archive path configuration = instrument_server.InstrumentConfiguration() configuration.instrument_short_name = parser.values.instrument session.proposal_id = parser.values.proposal session.config = configuration session.get_archive_tree() # get the file tuples (local name, archive name) to bundle tuples = session.files.get_bundle_files(parser.values.file_list) tasks.upload_files( bundle_name=parser.values.bundle_name, instrument_name=parser.values.instrument, proposal=parser.values.proposal, file_list=tuples, bundle_size=session.files.bundle_size, groups=parser.groups, server=parser.values.server, user=parser.values.user, password=parser.values.password)
session.bundle_filepath = os.path.join(configuration.target_dir, session.current_time + ".tar") meta_list = metadata.create_meta_upload_list() # spin this off as a background process and load the status page if home.task_comm.USE_CELERY: session.bundle_process = \ tasks.upload_files.delay(ingest_server=configuration.ingest_server, bundle_name=session.bundle_filepath, file_list=tuples, bundle_size=session.files.bundle_size, meta_list=meta_list) else: # run local tasks.upload_files(ingest_server=configuration.ingest_server, bundle_name=session.bundle_filepath, file_list=tuples, bundle_size=session.files.bundle_size, meta_list=meta_list) except Exception, e: session.is_uploading = False return HttpResponseServerError(json.dumps(e.message), content_type="application/json") return HttpResponse(json.dumps("success"), content_type="application/json") def upload_files(request): """ view for upload process spawn """ try: # use this flag to determine status of upload in incremental status session.is_uploading = True
def upload_from_options(parser): """ Upload files based upon command line options supecified in an OptionParser """ configdir = parser.values.config_dir # defaults for missing command line args configuration = instrument_server.UploaderConfiguration() configuration.initialize_settings(configdir) # populate metadata. Command line arguments override hard-coded config file arguments metadata = QueryMetadata.QueryMetadata(configuration.policy_server) metadata.load_meta(configdir) check_options(parser, configuration, metadata) # don't clean tar directory tar_man.CLEAN_TAR = False # typically the user of record is picked from a list based on proposal # here, we fetch a specific user from the User table node = metadata.get_node('EmslUserOfRecord') if parser.values.userOfRecord == '': network_user = node.value else: network_user = parser.values.userOfRecord user_record = metadata.get_Pacifica_user(network_user) node.value = user_record['person_id'] node = metadata.get_node('logon') if parser.values.user == '': network_user = node.value else: network_user = parser.values.user node.value = user_record['person_id'] node = metadata.get_node('instrumentByID') if parser.values.instrument == '': parser.values.instrument = node.value else: node.value = parser.values.instrument node = metadata.get_node('ProposalByInstrument') if parser.values.proposal == '': parser.values.proposal = node.value else: node.value = parser.values.proposal # populate so that we are running the same process as the # django uploader file_manager = file_tools.FileManager() # get rid of redundant file paths files = file_manager.filter_selected_list(parser.values.file_list) file_manager.data_dir = parser.values.work_dir if file_manager.data_dir == '': file_manager.data_dir = configuration.data_dir # add a final separator file_manager.common_path = os.path.join(parser.values.work_dir, '') tartar = False if parser.values.tartar == 'True': tartar = True # get the file tuples (local name, archive name) to bundle tuples = file_manager.get_bundle_files(files) meta_list = metadata.create_meta_upload_list() # let the task know to simply update and print the state task_comm.TaskComm.USE_CELERY = False # pylint: disable=unexpected-keyword-arg tasks.upload_files(ingest_server=configuration.ingest_server, bundle_name=parser.values.bundle_name, file_list=tuples, bundle_size=file_manager.bundle_size, meta_list=meta_list, auth=configuration.auth, verify=parser.values.verify, tartar=tartar)
if TaskComm.USE_CELERY: upload_process = \ tasks.upload_files.delay(ingest_server=configuration.ingest_server, bundle_name=bundle_filepath, file_list=tuples, bundle_size=file_manager.bundle_size, meta_list=meta_list, auth=configuration.auth, verify=configuration.verify) request.session['upload_process'] = upload_process.task_id request.session.modified = True print 'setting process id to: ' + request.session['upload_process'] else: # run local tasks.upload_files(ingest_server=configuration.ingest_server, bundle_name=bundle_filepath, file_list=tuples, bundle_size=file_manager.bundle_size, meta_list=meta_list, auth=configuration.auth) except Exception, ex: set_uploading(request, False) return HttpResponseBadRequest(str(ex), content_type='application/json') return HttpResponse(json.dumps('success'), content_type='application/json') def upload_files(request): """ view for upload process spawn """ # use this flag to determine status of upload in incremental status