def __connect(self): """ Connect to the panoptes client api :return: """ Panoptes.connect(username=self.username, password=self.password) return Project.find(self.project_id)
def upload_manifest_to_galaxy_zoo(subject_set_name, manifest, galaxy_zoo_id='5733', n_processes=10): """ Save manifest (set of galaxies with metadata prepared) to Galaxy Zoo Args: subject_set_name (str): name for subject set manifest (list): containing dicts of form {png_loc: img.png, key_data: {metadata_col: metadata_value}} galaxy_zoo_id (str): panoptes project id e.g. '5733' for Galaxy Zoo, '6490' for mobile n_processes (int): number of processes with which to upload galaxies in parallel Returns: None """ if 'TEST' in subject_set_name: logging.warning('Testing mode detected - not uploading!') return manifest if galaxy_zoo_id == '5733': logging.info('Uploading to Galaxy Zoo project 5733') elif galaxy_zoo_id == '6490': logging.info('Uploading to mobile app project 6490') else: logging.info('Uploading to unknown project {}'.format(galaxy_zoo_id)) # Important - don't commit the password! zooniverse_login = read_data_from_txt(zooniverse_login_loc) Panoptes.connect(**zooniverse_login) galaxy_zoo = Project.find(galaxy_zoo_id) subject_set = SubjectSet() subject_set.links.project = galaxy_zoo subject_set.display_name = subject_set_name subject_set.save() pbar = tqdm(total=len(manifest), unit=' subjects uploaded') save_subject_params = {'project': galaxy_zoo, 'pbar': pbar} save_subject_partial = functools.partial(save_subject, **save_subject_params) pool = ThreadPool(n_processes) new_subjects = pool.map(save_subject_partial, manifest) pbar.close() pool.close() pool.join() # new_subjects = [] # for subject in manifest: # print(subject) # new_subjects.append(save_subject_partial(subject)) subject_set.add(new_subjects) return manifest # for debugging only
def download(project_id, output, generate, generate_timeout, data_type): project = Project.find(project_id) export = project.get_export( data_type, generate=generate, wait_timeout=generate_timeout ) for chunk in export.iter_content(): output.write(chunk)
def create(self, request, *args, **kwargs): serializer = self.get_serializer(data=request.data) serializer.is_valid(raise_exception=True) with get_authenticated_panoptes(request.session['bearer_token'], request.session['bearer_expiry']): try: Project.find(_id_for(request.data['project'])) self.perform_create(serializer) headers = self.get_success_headers(serializer.data) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) except PanoptesAPIException: return Response(serializer.data, status=status.HTTP_401_UNAUTHORIZED, headers=headers)
def _create_subject_set(self, project_id, subject_set_name): project = Project.find(project_id) subject_set = SubjectSet() subject_set.display_name = subject_set_name subject_set.links.project = project subject_set.save() return subject_set
def delete(force, project_ids): for project_id in project_ids: project = Project.find(project_id) if not force: click.confirm( 'Delete project {} ({})?'.format( project_id, project.display_name, ), abort=True, ) project.delete()
def _create_subject(self, project_id, filename, metadata=None): subject = Subject() subject.links.project = Project.find(project_id) subject.add_location(filename) if metadata: subject.metadata.update(metadata) subject.save() return subject
def modify(project_id, display_name, description, primary_language, private): project = Project.find(project_id) if display_name: project.display_name = display_name if description: project.description = description if primary_language: project.primary_language = primary_language if private is not None: project.private = private project.save() echo_project(project)
def main(production=False): uname = input('Enter your username: '******'https://panoptes-staging.zooniverse.org', admin=True ) pId = 5733 # if production else 1820 project = Project.find(pId) subject_set = SubjectSet() subject_set.links.project = project subject_set.display_name = 'Test_subject_set_' + str(int(time.time())) subject_set.save() loc = os.path.abspath(os.path.dirname(__file__)) subjects = os.listdir(loc + '/subjects') images, differences, model, metadata = [ sorted(( int(re.match(r'{}_([0-9]+)\.(?:json|png)$'.format(s), i).group(1)) for i in subjects if re.match(r'{}_([0-9]+)\.(?:json|png)$'.format(s), i) )) for s in ('difference', 'image', 'model', 'metadata') ] if not images == differences == model == metadata: print( 'Images, differences, model and metadata ' + 'must all have same length' ) # TODO: change subject directory structure to be more efficient # (not having 12,000+ files in a folder...) for i in images: try: with open('{}/subjects/metadata_{}.json'.format(loc, i)) as f: metadata = json.load(f) except IOError: metadata = {} subject_set = uploadSubjectToSet( project, subject_set, [[j.format(loc, i) for j in ( '{}/subjects/image_{}.png', '{}/subjects/difference_{}.json', '{}/subjects/model_{}.json' )]], # locations [metadata], )
def create_subject_set(folder_name, set_name='test_subject_set'): subject_names = [ i.group(1) for i in ( re.match(r'image_(.*?).png', f) for f in os.listdir(folder_name) ) if i is not None ] files = [ [ join(folder_name, file_name) for file_name in ( 'image_{}.png'.format(subject_name), 'difference_{}.json'.format(subject_name), 'model_{}.json'.format(subject_name), 'metadata_{}.json'.format(subject_name), ) ] for subject_name in subject_names ] assert all(os.path.exists(j) for i in files for j in i), 'Missing files!' uname = input('Enter your username: ') pwd = getpass.getpass() Panoptes.connect( username=uname, password=pwd, admin=True ) pId = 5590 project = Project.find(pId) subject_set = SubjectSet() subject_set.links.project = project subject_set.display_name = set_name subject_set.save() metadata_list = [] for fs in files: try: with open(fs[3]) as metaF: metadata = json.load(metaF) except IOError: metadata = {} metadata_list.append(metadata) subject_set = uploadSubjectToSet( project, subject_set, [i[:3] for i in files], metadata_list, )
def __init__(self, zoo_project_id): self.zoo_project_id = zoo_project_id tmp = Project.find(zoo_project_id) self.project_info = flatten(tmp.raw) # Determine workflow order self.workflow_info = {} order = self.project_info['configuration_workflow_order'] workflows = [int(str(iWorkflow)) for iWorkflow in order] self.workflow_order = workflows # Save workflow information for iWorkflow in workflows: tmp1 = Workflow.find(iWorkflow) self.workflow_info[str(iWorkflow)] = flatten(tmp1.raw)
def modify(project_id, display_name, description, primary_language, public): """ Changes the attributes of an existing project. Any attributes which are not specified are left unchanged. """ project = Project.find(project_id) if display_name: project.display_name = display_name if description: project.description = description if primary_language: project.primary_language = primary_language if public is not None: project.private = not public project.save() echo_project(project)
def upload_images(id, use_database=True): print('Create subject set and upload images for', id) if use_database: update_status(id, gz_status='Uploading') wd = os.getcwd() Panoptes.connect(username='******', password=os.environ['PANOPTES_PASSWORD']) os.chdir(target + id) project = Project.find(slug='chrismrp/radio-galaxy-zoo-lofar') subject_set = SubjectSet() subject_set.display_name = id subject_set.links.project = project subject_set.save() print('Made subject set') new_subjects = [] g = glob.glob('*-manifest.txt') for i, f in enumerate(g): bits = open(f).readlines()[0].split(',') metadata = { 'subject_id': int(bits[0]), 'ra': float(bits[5]), 'dec': float(bits[6]), '#size': float(bits[7]), 'source_name': bits[4] } print('Upload doing', bits[4], '%i/%i' % (i, len(g))) subject = Subject() subject.links.project = project subject.metadata.update(metadata) for location in bits[1:4]: subject.add_location(location) subject.save() new_subjects.append(subject) subject_set.add(new_subjects) workflow = Workflow(11973) workflow.links.subject_sets.add(subject_set) if use_database: update_status(id, gz_status='In progress') print('Done!')
def main(production=False): uname = input('Enter your username: '******'https://panoptes-staging.zooniverse.org', admin=True) pId = 5590 if production else 1820 project = Project.find(pId) subject_set = SubjectSet() subject_set.links.project = project subject_set.display_name = 'Test_subject_set_' + str(int(time.time())) subject_set.save() loc = os.path.abspath(os.path.dirname(__file__)) subjects = os.listdir(loc + '/subjects') # TODO: change subject directory structure to be more efficient # (not having 12,000+ files in a folder...) for i in range(20): if 'image_{}.png'.format(i) in subjects: try: with open('{}/subjects/metadata_{}.json'.format(loc, i)) as f: metadata = json.load(f) except IOError: metadata = {} subject_set = uploadSubjectToSet( project, subject_set, [[ j.format(loc, i) for j in ('{}/subjects/image_{}.png', '{}/subjects/difference_{}.json', '{}/subjects/model_{}.json') ]], # locations [metadata], ) else: break
def create_subjects_and_link_to_project(self, proto_subjects, project_id, workflow_id, subject_set_id): try: USERNAME = os.getenv('PANOPTES_USERNAME') PASSWORD = os.getenv('PANOPTES_PASSWORD') Panoptes.connect(username=USERNAME, password=PASSWORD, endpoint=self.ENDPOINT) project = Project.find(project_id) workflow = Workflow().find(workflow_id) if subject_set_id == None: subject_set = SubjectSet() ts = time.gmtime() subject_set.display_name = time.strftime( "%m-%d-%Y %H:%M:%S", ts) subject_set.links.project = project subject_set.save() else: subject_set = SubjectSet().find(subject_set_id) subjects = [] for proto_subject in proto_subjects: subject = Subject() subject.links.project = project subject.add_location(proto_subject['location_lc']) subject.add_location(proto_subject['location_ps']) subject.metadata.update(proto_subject['metadata']) subject.save() subjects.append(subject) subject_set.add(subjects) workflow.add_subject_sets(subject_set) except Exception: self.log.exception("Error in create_subjects_and_link_to_project ")
def get_conn(self): if self._panoptes_client is None: self.log.info(f"{self.__class__.__name__} version {__version__}") self.log.debug( f"getting connection information from {self._conn_id}") config = self.get_connection(self._conn_id) ctyp = config.conn_type or self.DEFAULT_CONN_TYPE host = config.host or self.DEFAULT_HOST port = config.port or self.DEFAULT_PORT slug = config.schema login = config.login password = config.password if config.extra: try: extra = json.loads(config.extra) except json.decoder.JSONDecodeError: self._auto_disable_subject_sets = False else: self._auto_disable_subject_sets = extra.get( "auto_disable_ssets", False) if not login: raise MissingLoginError(self._conn_id) if not password: raise MissingPasswordError(self._conn_id) if not slug: raise MissingSchemaError(self._conn_id) project_slug = f"{login}/{slug}" endpoint = f"{ctyp}://{host}:{port}" self._panoptes_client = Panoptes.connect(username=login, password=password, endpoint=endpoint) self._project = Project.find(slug=project_slug) self.log.info( f"Searching project by slug {project_slug} found: {self._project}" ) return self._panoptes_client, self._project
def test_find_slug(self): p = Project.find(slug='zooniverse/snapshot-supernova') self.assertEqual(p.id, '1')
def test_find_id(self): p = Project.find(1) self.assertEqual(p.id, '1')
import os import io import csv from PIL import Image import numpy as np import cv2 as cv import pytesseract from datetime import datetime import panoptes_client from panoptes_client import SubjectSet, Project, Panoptes import requests Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) project = Project.find(slug='watchablewildlife/nebraska-spotted-skunk-project') def text_to_date(in_text): # this function reformats and tests text that has been read by tesseract try: split_text = in_text.split(' ') month = { 'JAN': '01', 'FEB': '02', 'MAR': '03', 'APR': '04', 'MAY': '05', 'JUN': '06', 'JUL': '07', 'AUG': '08',
else: choice = 'Q1' print('Input did not match expect options') get_out = input('Do you want to exit? "y" or "n"' + '\n') if get_out.lower() == 'y': quit() step_to_analyse = choice.upper() base_subject_set_id = '15195' print('The default base data set is ', base_subject_set_id) choice = input('Press "enter" to accept default, or enter another subject_set_id' + '\n') if choice != '': base_subject_set_id = str(choice) Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) proj = Project.find(slug='tedcheese/snapshots-at-sea') try: # check if the subject set already exits subj_set = SubjectSet.where(project_id=proj.id, subject_set_id=base_subject_set_id).next() print("Found base data set: {}.".format(subj_set.display_name)) base_data_set = subj_set.display_name except: base_data_set = '' print('base data subject set not found') quit() advance_steps = {'Q1': [427, base_data_set.partition(' ')[0] + '_Q2_' + base_data_set.partition(' ')[2], 'least one', 5, .75], 'Q2': [433, base_data_set.partition(' ')[0] + '_Q3_' + base_data_set.partition(' ')[2], 'any whales', 5, .75], 'Q3': [505, base_data_set.partition(' ')[0] + '_Q4_' + base_data_set.partition(' ')[2],
'APA Citation': apa_citation, 'Date': item_date, 'Library Of Congress Item ID': loc_id, 'Source Collection': source_collection, 'Title': item_title } segments.append(segment) print('Item segments transformation complete.') return segments segments = transform_item_segments('https://www.loc.gov/item/' + LIBRARY_OF_CONGRESS_ITEM_ID) Panoptes.connect(username=USERNAME, password=PASSWORD, endpoint=ENDPOINT) project = Project.find(PROJECT) subject_set = SubjectSet() subject_set.links.project = project subject_set.display_name = segments[0]['metadata']['Title'] # uses item Title as default subject set name, or feel free to hardcode subject_set.save() print('Begin Zooniverse subject upload...') for segment in segments: subject = Subject() subject.links.project = project subject.add_location(segment['location']) subject.metadata.update(segment['metadata'])
def test_find_unknown_id(self): p = Project.find(0) self.assertEqual(p, None)
phone : +32 (0)2 373.04.19 e-mail : [email protected] web : www.aeronomie.be ________________________________________________ """ filename = "panoptes_test2_export.csv" from panoptes_client import Project, Panoptes from panoptes_client.panoptes import PanoptesAPIException import requests, sys Panoptes.connect(username=username, password=password) project = Project.find(slug='zooniverse/radio-meteor-zoo') #r = project.get_classifications_export(generate=True, wait=True, wait_timeout=1800) wait_timeout = 60 project.generate_classifications_export() for attempt in range(60): print "wait classification export (attempt %d)" % attempt sys.stdout.flush() try: export = project.wait_classifications_export(wait_timeout) except PanoptesAPIException as e: print str(e)[:32] if str(e)[:32] == "classifications_export not ready": continue else: raise
""" This version is written in Python 3.66 This script adds a link to the comments in a subject's metadata which oopens in a new tab. This allows one to comment while classifying. It requires the project owner credentials to be set up as OS environmental variables, and an appropriate project slug modified on line 12. """ import os from PIL import Image, ExifTags import panoptes_client from panoptes_client import SubjectSet, Project, Panoptes import requests Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) project_slug = 'insert project slug here' project = Project.find(slug=project_slug) while True: set_id = input('Entry subject set id to modify:' + '\n') try: subject_set = SubjectSet.find(set_id) count_subjects = 0 subject_list = [] for subject in subject_set.subjects: count_subjects += 1 link = r'[Comments](+tab+https://www.zooniverse.org/projects/' \ + project_slug + r'/talk/subjects/' + str(subject.id) + ')' subject.metadata['Talk Pages'] = link print(subject.id, subject.metadata['Talk Pages'], ' added') subject.save() break except panoptes_client.panoptes.PanoptesAPIException:
""" This version is written in Python 3.62""" import os from panoptes_client import SubjectSet, Project, Panoptes # edit this line for your directory path for where you want the listing to go: directory_path = r'C:\py\Data_digging\Panoptes_client' # set up your zooniverse sign-in credentials as environment variables: Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) # edit the project slug for your project: project = Project.find(slug='rainforestexpeditions/amazon-aerobotany') while True: set_name = input('Entry a name for the subject set to use' + '\n') try: # check if the subject set already exits subject_set = SubjectSet.where(project_id=project.id, display_name=set_name).next() retry = input( 'Subject set found, Enter "n" to cancel this quiry, any other key to continue' + '\n') if retry.lower() == 'n': quit() break except StopIteration: retry = input( 'Subject set not found, Enter "n" to cancel, any other key to try again' + '\n') if retry.lower() == 'n':
else: print('That entry is not a valid path for an existing directory') retry = input('Enter "y" to try again, any other key to exit' + '\n') if retry.lower() != 'y': quit() print('Entry a name for the subject set to use or create.') set_name = input( 'This will also be used as the Site-Date in the subject metadata:' + '\n') # connect to zooniverse using a user_name and password previously stored as environmental variables in the user's OS Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) # OR # Panoptes.connect(username='******', password='******') # NOT secure!!!! project = Project.find(slug='pmason/fossiltrainer') # project slug # needs to be changed for different projects file_list = get_files_in(location) # go get the file list number = len(file_list) previous_subjects = [] try: # check if the subject set already exits: subject_set = SubjectSet.where(project_id=project.id, display_name=set_name).next() print('You have chosen to upload ', number, ' videos to an existing subject set', set_name) retry = input( 'Enter "n" to cancel this upload, any other key to continue' + '\n') if retry.lower() == 'n': quit()
3) Sky match the tables by RA and DEC in topcat, with the MaNGA data as table 1, and GZ as table 2. 4) Remove the second set of RA and DEC columns, and name the first ones 'RA' and 'DEC' 5) Save this file as a csv 6) This only works in python 2 because it uses panoptes_client ''' import numpy as np from panoptes_client import SubjectSet, Subject, Project, Panoptes import os import progressbar as pb myusername = os.environ['PANOPTES_USERNAME'] mypassword = os.environ['PANOPTES_PASSWORD'] Panoptes.connect(username= myusername, password=mypassword) project = Project.find(id='73') fullsample = SubjectSet.find(5326) spirals = SubjectSet.find(5324) bars = SubjectSet.find(5325) progress = pb.ProgressBar(widgets=[pb.Bar(), pb.ETA()]) data = np.genfromtxt('../GZ3D/MatchedData.csv', delimiter = ',', names=True, dtype=[('DEC', float), ('IAUNAME', '|S30'),('IFUTARGETSIZE',int), ('MANGAID', '|S10'),('MANGA_TILEID',int),('NSAID', int), ('PETROTH50',float),('RA',float),('SERSIC_TH50',float), ('Z',float),('specobjid', int),('dr8objid', int), ('dr7objid', int),('t01_smooth_or_features_a02_features_or_disk_weighted_fraction', float), ('t02_edgeon_a05_no_weighted_fraction', float), ('t03_bar_a06_bar_weighted_fraction', float), ('t04_spiral_a08_spiral_weighted_fraction', float)]) counter = 0
# Read filenames to upload # import pdb; pdb.set_trace() # files = sorted(glob.glob(path1+'*.jpeg')) # files = files + sorted(glob.glob(path2+'*.jpeg')) # files = files + sorted(glob.glob(path3+'*C02*.jpeg')) files = sorted(glob.glob(path1 + '*[02468]??.jpeg')) # Create metadata subject_metadata = {} for f, file in enumerate(files): subject_metadata[file] = {'file': file, 'subject_reference': f} Panoptes.connect(username=username, password=password) # tutorial_project = Project() tutorial_project = Project.find(7699) # tutorial_project.display_name = display_name # tutorial_project.description = description # tutorial_project.primary_language = 'en' # tutorial_project.private =True # tutorial_project.save() subject_set = SubjectSet() subject_set.links.project = tutorial_project subject_set.display_name = subject_name subject_set.save() tutorial_project.reload() print(tutorial_project.links.subject_sets) new_subjects = []
def info(project_id): project = Project.find(project_id) click.echo(yaml.dump(project.raw))
.5) except KeyError: avg_act = line[4] try: build_part += "{:<12},{:<13},{:<16},{:<25},{:<22},{:<19},{:<21},{:<19},{:<21},{:<22},{}".format( 'Project_id', 'Workflow_id', 'Subjects_count', 'Retired_subjects_count', 'Classifications_count', 'Project activity', '14 day avg. activity', 'Retirement_limit', 'Class_Completeness', 'Retired_Completeness', 'Linearized completeness', 'Display_name') + '\n' build_overall = "{:<12},{:<13},{:<16},{:<25},{:<22},{:<19},{:<21},{:<19},{:<21},{:<22},{:<24},{}".format( str(line[0]), 'Over_all', str(line[1]), str(line[2]), str(line[3]), str(line[4]), str(avg_act), '', '', str(line[5]), '', json.dumps(str(line[7]), ensure_ascii=False)) + '\n' project = Project.find(str(line[0])) for wrkflw in project.links.workflows: if wrkflw.active == 1: if wrkflw.retirement['criteria'] == 'classification_count': try: ret_limit = wrkflw.retirement['options']['count'] if wrkflw.classifications_count < ( wrkflw.subjects_count * ret_limit): class_comp = int( wrkflw.classifications_count / (wrkflw.subjects_count * wrkflw.retirement['options']['count']) * 100 + .5) else: class_comp = 100 except KeyError:
subject_set_old = '' retry = input('Enter "y" to try again, any other key to exit' + '\n') if retry.lower() != 'y': quit() # build list of subjects in source subject set add_subjects = [] for subject in subject_set_old.subjects: add_subjects.append(subject.id) print(len(add_subjects), ' subjects found in the subject set to copy') # get project id of destination project; while True: proj_id = input('Enter the project id for the new subject set:' + '\n') try: proj = Project.find(proj_id) break except: print('Project not found or accessible') retry = input('Enter "y" to try again, any other key to exit' + '\n') if retry.lower() != 'y': quit() # get new subject name new_set_name = input('Enter a name for the subject set to use or create:' + '\n') # find or build destination subject set try: # check if the subject set already exits subject_set_new = SubjectSet.where(project_id=proj.id, display_name=new_set_name).next() except:
# ensure the file pointer is returned to the beginning of the file-like object file_bytes.seek(0, 0) return file_bytes parser = argparse.ArgumentParser(description='Zooniverse Uploader') parser.add_argument('image_dir') parser.add_argument('--subject', '-s', required=True) args = parser.parse_args() set_name = args.subject # connect to zooniverse - requires the User_name and Password to be set up as environmental variables in your OS Panoptes.connect(username=os.environ['ZOONIVERSE_USERNAME'], password=os.environ['ZOONIVERSE_PASSWORD']) # modify the project slug if used for other than Snapshots at Sea project = Project.find(slug='tedcheese/snapshots-at-sea') if not os.path.exists(args.image_dir): print('[%s] does not exist.' % args.image_dir) sys.exit() # load the list of image files found in the directory: # The local file name will be uploaded as metadata with the image file_types = ['jpg', 'jpeg'] subject_metadata = {} for entry in os.listdir(args.image_dir): if entry.partition('.')[2].lower() in file_types: subject_metadata[entry] = {'Filename': entry} print('Found ', len(subject_metadata), ' files to upload in this directory.') previous_subjects = []
""" This version is written in Python 3.66 This script attempts to retrieve the exif data from existing subject image files and add the datetime to the subject metadata. It requires the project owner credentials to be set up as OS environmental variables, and an appropriate project slug modified on line 11. depending on the camera used to take the original subject image the exif code may be different than that in the code and may need to be modified""" import os from PIL import Image, ExifTags import panoptes_client from panoptes_client import SubjectSet, Project, Panoptes import requests Panoptes.connect(username=os.environ['User_name'], password=os.environ['Password']) project = Project.find(slug='pmason\fossiltrainer') while True: set_id = input('Entry subject set id to update:' + '\n') try: subject_set = SubjectSet.find(set_id) count_subjects = 0 subject_list = [] for subject in subject_set.subjects: count_subjects += 1 if subject.metadata['DateTime'] == '': try: img = Image.open(requests.get(subject.locations[0]['image/jpeg'], stream=True).raw) exif_dict = img._getexif() date_time = exif_dict[306] except (IOError, KeyError): print('Acquiring exif data for ', subject.id, ' failed') continue subject.metadata['DateTime'] = date_time
def test_find_unknown_slug(self): with self.assertRaises(PanoptesAPIException): Project.find(slug='invalid_slug')