def ls(subject_set_id, project_id, workflow_id, quiet): """Lists subject set IDs and names""" if subject_set_id and not project_id and not workflow_id: subject_set = SubjectSet.find(subject_set_id) if quiet: click.echo(subject_set.id) else: echo_subject_set(subject_set) return args = {} if project_id: args['project_id'] = project_id if workflow_id: args['workflow_id'] = workflow_id if subject_set_id: args['subject_set_id'] = subject_set_id subject_sets = SubjectSet.where(**args) if quiet: click.echo(" ".join([s.id for s in subject_sets])) else: for subject_set in subject_sets: echo_subject_set(subject_set)
import sys from panoptes_client import SubjectSet, Project, Panoptes csv.field_size_limit(sys.maxsize) # connect to Panoptes_Client Note Os environment variables must be set! Else use the commented # out line with your user_name and password hardcoded (in this case keep the code secure!). # Panoptes.connect(username='******', password='******') Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) # get source project id and existing subject set name while True: proj_id = input('Enter the project id for the subject set to copy:' + '\n') set_name = input('Enter a name for the subject set to copy:' + '\n') try: subject_set_old = SubjectSet.where(project_id=proj_id, display_name=set_name).next() break except: print('subject set not found or not accessible') subject_set_old = '' retry = input('Enter "y" to try again, any other key to exit' + '\n') if retry.lower() != 'y': quit() # build list of subjects in source subject set add_subjects = [] for subject in subject_set_old.subjects: add_subjects.append(subject.id) print(len(add_subjects), ' subjects found in the subject set to copy') # get project id of destination project;
get_out = input('Do you want to exit? "y" or "n"' + '\n') if get_out.lower() == 'y': quit() step_to_analyse = choice.upper() base_subject_set_id = '15195' print('The default base data set is ', base_subject_set_id) choice = input('Press "enter" to accept default, or enter another subject_set_id' + '\n') if choice != '': base_subject_set_id = str(choice) Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) proj = Project.find(slug='tedcheese/snapshots-at-sea') try: # check if the subject set already exits subj_set = SubjectSet.where(project_id=proj.id, subject_set_id=base_subject_set_id).next() print("Found base data set: {}.".format(subj_set.display_name)) base_data_set = subj_set.display_name except: base_data_set = '' print('base data subject set not found') quit() advance_steps = {'Q1': [427, base_data_set.partition(' ')[0] + '_Q2_' + base_data_set.partition(' ')[2], 'least one', 5, .75], 'Q2': [433, base_data_set.partition(' ')[0] + '_Q3_' + base_data_set.partition(' ')[2], 'any whales', 5, .75], 'Q3': [505, base_data_set.partition(' ')[0] + '_Q4_' + base_data_set.partition(' ')[2], 'a fluke', 5, .45], 'Q4': [506, 'SAS_' + base_data_set, 'Humpback', 5, .5], }
set_name = input( 'This will also be used as the Site-Date in the subject metadata:' + '\n') # connect to zooniverse using a user_name and password previously stored as environmental variables in the user's OS Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) # OR # Panoptes.connect(username='******', password='******') # NOT secure!!!! project = Project.find(slug='pmason/fossiltrainer') # project slug # needs to be changed for different projects file_list = get_files_in(location) # go get the file list number = len(file_list) previous_subjects = [] try: # check if the subject set already exits: subject_set = SubjectSet.where(project_id=project.id, display_name=set_name).next() print('You have chosen to upload ', number, ' videos to an existing subject set', set_name) retry = input( 'Enter "n" to cancel this upload, any other key to continue' + '\n') if retry.lower() == 'n': quit() print( 'Checking for previously uploaded subjects, this could take a while!') for subject in subject_set.subjects: # if the subject set does exist get a listing of what is in it previous_subjects.append(subject.metadata['Filename']) except StopIteration: print('You have chosen to upload ', number, ' videos to an new subject set ', set_name) retry = input( 'Enter "n" to cancel this upload, any other key to continue' + '\n')
def get_subject_set(project_id: int, name: str): # will fail if duplicate display name - don't do this (not allowed, perhaps) try: return next(SubjectSet.where(project_id=project_id, display_name=name)) except StopIteration: raise ValueError(f'Project {project_id} has no subject set {name}')
def upload_manifest_to_galaxy_zoo( subject_set_name, manifest, project_id='5733', # default to main GZ project login_loc='zooniverse_login.txt'): """ Save manifest (set of galaxies with metadata prepared) to Galaxy Zoo Args: subject_set_name (str): name for subject set manifest (list): containing dicts of form {png_loc: img.png, key_data: {metadata_col: metadata_value}} project_id (str): panoptes project id e.g. '5733' for Galaxy Zoo, '6490' for mobile n_processes (int): number of processes with which to upload galaxies in parallel Returns: None """ assert os.path.exists(login_loc) if 'TEST' in subject_set_name: logging.warning('Testing mode detected - not uploading!') return manifest if project_id == '5733': logging.info('Uploading to Galaxy Zoo project 5733') elif project_id == '6490': logging.info('Uploading to mobile app project 6490') elif project_id == '8751': logging.info('Uploading to staging project 8751') else: logging.info('Uploading to unknown project {}'.format(project_id)) # Important - don't commit the password! zooniverse_login = read_data_from_txt(login_loc) Panoptes.connect(**zooniverse_login) project = Project.find(project_id) # check if subject set already exists subject_set = None subject_sets = SubjectSet.where(project_id=project_id) for candidate_subject_set in subject_sets: if candidate_subject_set.raw['display_name'] == subject_set_name: # use if it already exists subject_set = candidate_subject_set if not subject_set: # make a new one if not subject_set = SubjectSet() subject_set.links.project = project subject_set.display_name = subject_set_name subject_set.save() pbar = tqdm(total=len(manifest), unit=' subjects uploaded') save_subject_params = {'project': project, 'pbar': pbar} save_subject_partial = functools.partial(save_subject, **save_subject_params) # upload in async blocks, to avoid huge join at end manifest_block_start = 0 manifest_block_size = 100 while True: manifest_block = manifest[manifest_block_start:manifest_block_start + manifest_block_size] new_subjects = [] with Subject.async_saves(): for manifest_entry in manifest_block: new_subjects.append(save_subject_partial(manifest_entry)) subject_set.add(new_subjects) logging.info('{} subjects linked'.format(new_subjects)) manifest_block_start += manifest_block_size if manifest_block_start > len(manifest): break return manifest # for debugging only
image_set_name = 'image_set_' + month audio_set_name = 'audio_set_' + month # Conexión con Panoptes Panoptes.connect(username=user, password=passwd) # El proyecto "Sky Sounds" tiene asociado el identificador 13586. project = Project('13586') # ------- Subject set de imágenes ------- # Conexión con el subject set correspondiente o creación de uno nuevo en caso # de que este no exista. try: # Comprueba si existe el subject set. subject_set = SubjectSet.where(project_id=project.id, display_name=image_set_name).next() except StopIteration: # Crea un nuevo subject set para los nuevos datos y lo asocia al proyecto. subject_set = SubjectSet() subject_set.links.project = project subject_set.display_name = image_set_name subject_set.save() # Adicción de las muestras al subject set. with open(manifest_images_file, 'r') as mani_file: print('Uploading image_set') r = csv.DictReader(mani_file) for line in r: subject = Subject() subject.links.project = project