def main(sleep_time=constants.IMAGE_PROCESSOR_DAEMON_SLEEP_TIME_S):
logging.basicConfig(level=logging.INFO,
format=constants.LOG_FMT_S_THREADED)
logging.info("Reading images from " + config.GCS_BUCKET)
logging.info("Writing images from " + config.GCS_PROCESSED_PHOTOS_BUCKET)
#Get current projects storage and datastore client
credentials = sa.get_credentials()
datastore_client = datastore.Client(project=config.PROJECT_ID, \
credentials=credentials)
storage_client = storage.client.Client(project=config.PROJECT_ID, \
credentials=credentials)
# Create new instance of image_processor pipeline w/ datastore & GCS
image_processor_pipeline = pipeline.Pipeline(datastore_client, storage_client)
while True:
# Get all newly pre-processed images
fnames = image_processor_pipeline.scan(ds.DATASTORE_PHOTO)
if fnames:
processed_fnames = image_processor_pipeline.process(fnames)
if processed_fnames:
# copy the processed results to GCS, update the Photo record's
# processed field, and add the ProcessedImage record
uploaded_fnames = image_processor_pipeline.upload(processed_fnames)
# Allow files to accumulate before taking our next pass
time.sleep(sleep_time)
def main():
args = get_arguments()
credentials = sa.get_credentials()
client = datastore.Client(project=args.project_id, credentials=credentials)
query = client.query(kind="User")
query.keys_only()
entities = query.fetch()
for entity in entities:
print "User id (hashed):", entity.key.name
key = client.key("UserRole", entity.key.name)
entity = client.get(key)
print "\troles:", entity['roles']
def get_file_from_gcs(fname):
"""
Download all new files from GCS bucket w/ url <src> to destination folder.
Must be outside of pipeline class for use as multiprocess map worker
"""
storage_client = storage.client.Client(project=config.PROJECT_ID, \
credentials=sa.get_credentials())
try:
blob = storage_client.get_bucket(
config.GCS_PROCESSED_PHOTOS_BUCKET).get_blob(fname)
except Exception, e:
msg = 'Failed to download {0} from Cloud Storage.'
logging.exception(msg.format(fname))
return False
def _get_client(client_type='storage'):
"""
Returns gcloud client, (either for storage if `client_type` is 'storage',
or for datastore if `client_type` is 'datastore'). Defaults to storage
client, including when an invalid `client_type` is given.
Raises `CouldNotObtainCredentialsError` if there is an error obtaining
credentials.
"""
# Raises CouldNotObtainCredentialsError
credentials = sa.get_credentials()
if client_type == 'datastore':
client_class = datastore.Client
else:
client_class = storage.client.Client
return client_class(project=config.PROJECT_ID, credentials=credentials)
def get_file_from_gcs(fname):
"""
Download all new files from GCS bucket w/ url <src> to destination folder.
Must be outside of pipeline class for use as multiprocess map worker
"""
storage_client = storage.client.Client(project=config.PROJECT_ID, \
credentials=sa.get_credentials())
fpath = '{0}/{1}'.format(constants.MOVIE_DATA_DIR, fname)
if os.path.exists(fpath):
logging.info("Already downloaded file %s" % fpath)
return True
try:
blob = storage_client.get_bucket(
config.GCS_PROCESSED_PHOTOS_BUCKET).get_blob(fname)
except Exception, e:
msg = 'Failed to download {0} from Cloud Storage.'
logging.exception(msg.format(fname))
return False
def main(sleep_time=constants.MOVIE_DAEMON_SLEEP_TIME_S):
logging.basicConfig(level=logging.INFO,
format=constants.LOG_FMT_S_THREADED)
logging.info("Reading images from gs://" +
config.GCS_PROCESSED_PHOTOS_BUCKET)
logging.info("Writing movies to gs://" + config.GCS_MOVIE_BUCKET)
#Get current projects storage and datastore client
credentials = sa.get_credentials()
datastore_client = datastore.Client(project=config.PROJECT_ID, \
credentials=credentials)
storage_client = storage.client.Client(project=config.PROJECT_ID, \
credentials=credentials)
# Create new instance of movie pipeline w/ datastore & GCS
movie_pipeline = pipeline.Pipeline(datastore_client, storage_client)
movie_stats = pipeline_stats.Pipeline_Stats(datastore_client,
storage_client)
while True:
# Get all newly pre-processed images
fnames = movie_pipeline.scan()
# Stitch new images into new/exsisting megamovie(s)
if fnames:
# Seperate files based on clusters of photos
clusters = movie_stats.get_clusters(fnames)
# Concatenates frames into movies and stores in constants.MOVIE_DATA_DIR
files_in_movie = movie_pipeline.assemble(fnames)
# Upload movie file(s)
if files_in_movie:
movie_pipeline.upload(files_in_movie)
# Allow files to accumulate before taking our next pass
time.sleep(sleep_time)