def ls(subject_set_id, project_id, workflow_id, quiet):
"""Lists subject set IDs and names"""
if subject_set_id and not project_id and not workflow_id:
subject_set = SubjectSet.find(subject_set_id)
if quiet:
click.echo(subject_set.id)
else:
echo_subject_set(subject_set)
return
args = {}
if project_id:
args['project_id'] = project_id
if workflow_id:
args['workflow_id'] = workflow_id
if subject_set_id:
args['subject_set_id'] = subject_set_id
subject_sets = SubjectSet.where(**args)
if quiet:
click.echo(" ".join([s.id for s in subject_sets]))
else:
for subject_set in subject_sets:
echo_subject_set(subject_set)
import sys
from panoptes_client import SubjectSet, Project, Panoptes
csv.field_size_limit(sys.maxsize)
# connect to Panoptes_Client Note Os environment variables must be set! Else use the commented
# out line with your user_name and password hardcoded (in this case keep the code secure!).
# Panoptes.connect(username='******', password='******')
Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password'])
# get source project id and existing subject set name
while True:
proj_id = input('Enter the project id for the subject set to copy:' + '\n')
set_name = input('Enter a name for the subject set to copy:' + '\n')
try:
subject_set_old = SubjectSet.where(project_id=proj_id, display_name=set_name).next()
break
except:
print('subject set not found or not accessible')
subject_set_old = ''
retry = input('Enter "y" to try again, any other key to exit' + '\n')
if retry.lower() != 'y':
quit()
# build list of subjects in source subject set
add_subjects = []
for subject in subject_set_old.subjects:
add_subjects.append(subject.id)
print(len(add_subjects), ' subjects found in the subject set to copy')
# get project id of destination project;
get_out = input('Do you want to exit? "y" or "n"' + '\n')
if get_out.lower() == 'y':
quit()
step_to_analyse = choice.upper()
base_subject_set_id = '15195'
print('The default base data set is ', base_subject_set_id)
choice = input('Press "enter" to accept default, or enter another subject_set_id' + '\n')
if choice != '':
base_subject_set_id = str(choice)
Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password'])
proj = Project.find(slug='tedcheese/snapshots-at-sea')
try:
# check if the subject set already exits
subj_set = SubjectSet.where(project_id=proj.id, subject_set_id=base_subject_set_id).next()
print("Found base data set: {}.".format(subj_set.display_name))
base_data_set = subj_set.display_name
except:
base_data_set = ''
print('base data subject set not found')
quit()
advance_steps = {'Q1': [427, base_data_set.partition(' ')[0] + '_Q2_' + base_data_set.partition(' ')[2],
'least one', 5, .75],
'Q2': [433, base_data_set.partition(' ')[0] + '_Q3_' + base_data_set.partition(' ')[2],
'any whales', 5, .75],
'Q3': [505, base_data_set.partition(' ')[0] + '_Q4_' + base_data_set.partition(' ')[2],
'a fluke', 5, .45],
'Q4': [506, 'SAS_' + base_data_set, 'Humpback', 5, .5],
}
set_name = input(
'This will also be used as the Site-Date in the subject metadata:' + '\n')
# connect to zooniverse using a user_name and password previously stored as environmental variables in the user's OS
Panoptes.connect(username=os.environ['User_name'],
password=os.environ['Password']) # OR
# Panoptes.connect(username='******', password='******') # NOT secure!!!!
project = Project.find(slug='pmason/fossiltrainer') # project slug
# needs to be changed for different projects
file_list = get_files_in(location) # go get the file list
number = len(file_list)
previous_subjects = []
try:
# check if the subject set already exits:
subject_set = SubjectSet.where(project_id=project.id,
display_name=set_name).next()
print('You have chosen to upload ', number,
' videos to an existing subject set', set_name)
retry = input(
'Enter "n" to cancel this upload, any other key to continue' + '\n')
if retry.lower() == 'n':
quit()
print(
'Checking for previously uploaded subjects, this could take a while!')
for subject in subject_set.subjects: # if the subject set does exist get a listing of what is in it
previous_subjects.append(subject.metadata['Filename'])
except StopIteration:
print('You have chosen to upload ', number,
' videos to an new subject set ', set_name)
retry = input(
'Enter "n" to cancel this upload, any other key to continue' + '\n')
def get_subject_set(project_id: int, name: str):
# will fail if duplicate display name - don't do this (not allowed, perhaps)
try:
return next(SubjectSet.where(project_id=project_id, display_name=name))
except StopIteration:
raise ValueError(f'Project {project_id} has no subject set {name}')
def upload_manifest_to_galaxy_zoo(
subject_set_name,
manifest,
project_id='5733', # default to main GZ project
login_loc='zooniverse_login.txt'):
"""
Save manifest (set of galaxies with metadata prepared) to Galaxy Zoo
Args:
subject_set_name (str): name for subject set
manifest (list): containing dicts of form {png_loc: img.png, key_data: {metadata_col: metadata_value}}
project_id (str): panoptes project id e.g. '5733' for Galaxy Zoo, '6490' for mobile
n_processes (int): number of processes with which to upload galaxies in parallel
Returns:
None
"""
assert os.path.exists(login_loc)
if 'TEST' in subject_set_name:
logging.warning('Testing mode detected - not uploading!')
return manifest
if project_id == '5733':
logging.info('Uploading to Galaxy Zoo project 5733')
elif project_id == '6490':
logging.info('Uploading to mobile app project 6490')
elif project_id == '8751':
logging.info('Uploading to staging project 8751')
else:
logging.info('Uploading to unknown project {}'.format(project_id))
# Important - don't commit the password!
zooniverse_login = read_data_from_txt(login_loc)
Panoptes.connect(**zooniverse_login)
project = Project.find(project_id)
# check if subject set already exists
subject_set = None
subject_sets = SubjectSet.where(project_id=project_id)
for candidate_subject_set in subject_sets:
if candidate_subject_set.raw['display_name'] == subject_set_name:
# use if it already exists
subject_set = candidate_subject_set
if not subject_set: # make a new one if not
subject_set = SubjectSet()
subject_set.links.project = project
subject_set.display_name = subject_set_name
subject_set.save()
pbar = tqdm(total=len(manifest), unit=' subjects uploaded')
save_subject_params = {'project': project, 'pbar': pbar}
save_subject_partial = functools.partial(save_subject,
**save_subject_params)
# upload in async blocks, to avoid huge join at end
manifest_block_start = 0
manifest_block_size = 100
while True:
manifest_block = manifest[manifest_block_start:manifest_block_start +
manifest_block_size]
new_subjects = []
with Subject.async_saves():
for manifest_entry in manifest_block:
new_subjects.append(save_subject_partial(manifest_entry))
subject_set.add(new_subjects)
logging.info('{} subjects linked'.format(new_subjects))
manifest_block_start += manifest_block_size
if manifest_block_start > len(manifest):
break
return manifest # for debugging only
image_set_name = 'image_set_' + month
audio_set_name = 'audio_set_' + month
# Conexión con Panoptes
Panoptes.connect(username=user, password=passwd)
# El proyecto "Sky Sounds" tiene asociado el identificador 13586.
project = Project('13586')
# ------- Subject set de imágenes -------
# Conexión con el subject set correspondiente o creación de uno nuevo en caso
# de que este no exista.
try:
# Comprueba si existe el subject set.
subject_set = SubjectSet.where(project_id=project.id,
display_name=image_set_name).next()
except StopIteration:
# Crea un nuevo subject set para los nuevos datos y lo asocia al proyecto.
subject_set = SubjectSet()
subject_set.links.project = project
subject_set.display_name = image_set_name
subject_set.save()
# Adicción de las muestras al subject set.
with open(manifest_images_file, 'r') as mani_file:
print('Uploading image_set')
r = csv.DictReader(mani_file)
for line in r:
subject = Subject()
subject.links.project = project
