Ejemplo n.º 1
0
 def test09_delete_resources(self):
     """ should allow deleting resources """
     uploader.upload("tests/logo.png", public_id="api_test3")
     resource = api.resource("api_test3")
     self.assertNotEqual(resource, None)
     api.delete_resources(["apit_test", "api_test2", "api_test3"])
     self.assertRaises(api.NotFound, api.resource, ("api_test3"))
Ejemplo n.º 2
0
 def setUp(self):
     self.timestamp_tag = "api_test_tag_{0}".format(utils.now())
     if ApiTest.initialized: return
     ApiTest.initialized = True
     cloudinary.reset_config()
     if not cloudinary.config().api_secret: return
     try:
         api.delete_resources(["api_test", "api_test2", "api_test3"])
     except Exception:
         pass
     for transformation in ["api_test_transformation", "api_test_transformation2", "api_test_transformation3"]:
         try:
             api.delete_transformation(transformation)
         except Exception:
             pass
     for transformation in ["api_test_upload_preset", "api_test_upload_preset2", "api_test_upload_preset3",
                            "api_test_upload_preset4"]:
         try:
             api.delete_upload_preset(transformation)
         except Exception:
             pass
     
     for id in ["api_test", "api_test2"]:
         uploader.upload("tests/logo.png",
                         public_id=id, tags=["api_test_tag", self.timestamp_tag],
                         context="key=value", eager=[{"width": 100, "crop": "scale"}])
Ejemplo n.º 3
0
 def setUp(self):
     self.timestamp_tag = "api_test_tag_{0}".format(utils.now())
     if ApiTest.initialized: return
     ApiTest.initialized = True
     cloudinary.reset_config()
     if not cloudinary.config().api_secret: return
     try:
         api.delete_resources(["api_test", "api_test2", "api_test3"])
     except Exception:
         pass
     for transformation in ["api_test_transformation", "api_test_transformation2", "api_test_transformation3"]:
         try:
             api.delete_transformation(transformation)
         except Exception:
             pass
     for transformation in ["api_test_upload_preset", "api_test_upload_preset2", "api_test_upload_preset3",
                            "api_test_upload_preset4"]:
         try:
             api.delete_upload_preset(transformation)
         except Exception:
             pass
     
     for id in ["api_test", "api_test2"]:
         uploader.upload("tests/logo.png",
                         public_id=id, tags=["api_test_tag", self.timestamp_tag],
                         context="key=value", eager=[{"width": 100, "crop": "scale"}])
Ejemplo n.º 4
0
 def test09_delete_resources(self):
     """ should allow deleting resources """
     uploader.upload("tests/logo.png", public_id="api_test3")
     resource = api.resource("api_test3")
     self.assertNotEqual(resource, None)
     api.delete_resources(["apit_test", "api_test2", "api_test3"])
     self.assertRaises(api.NotFound, api.resource, ("api_test3"))
Ejemplo n.º 5
0
    def tearDownClass(cls):
        try:
            api.delete_resources([API_TEST_ID, API_TEST_ID2, API_TEST_ID3, API_TEST_ID4, API_TEST_ID5])
        except Exception:
            pass
        for transformation in [API_TEST_TRANS, API_TEST_TRANS2, API_TEST_TRANS3, API_TEST_TRANS_SCALE100_STR]:
            try:
                api.delete_transformation(transformation)
            except Exception:
                pass
        presets_response = api.upload_presets(max_results=200)
        preset_names = [
            preset["name"] for preset in presets_response.get('presets', [])
            if UNIQUE_API_TAG in preset.get('settings', {}).get('tags', '')]
        for name in preset_names:
            try:
                api.delete_upload_preset(name)
            except Exception:
                pass
        cloudinary.api.delete_resources_by_tag(UNIQUE_API_TAG)
        cloudinary.api.delete_resources_by_tag(UNIQUE_API_TAG, resource_type='raw')

        try:
            api.delete_upload_mapping(MAPPING_TEST_ID)
        except Exception:
            pass
Ejemplo n.º 6
0
    def test09c_delete_resources_by_transformations(self, mocker):
        """ should allow deleting resources by transformations """
        mocker.return_value = MOCK_RESPONSE

        api.delete_resources(['api_test', 'api_test2'],
                             transformations=['c_crop,w_100'])
        self.assertEqual(get_method(mocker), 'DELETE')
        self.assertEqual(get_param(mocker, 'transformations'), 'c_crop,w_100')

        api.delete_all_resources(
            transformations=['c_crop,w_100', {
                "crop": "scale",
                "width": 107
            }])
        self.assertEqual(get_method(mocker), 'DELETE')
        self.assertEqual(get_param(mocker, 'transformations'),
                         'c_crop,w_100|c_scale,w_107')

        api.delete_resources_by_prefix("api_test_by",
                                       transformations='c_crop,w_100')
        self.assertEqual(get_method(mocker), 'DELETE')
        self.assertEqual(get_param(mocker, 'transformations'), 'c_crop,w_100')

        api.delete_resources_by_tag("api_test_tag",
                                    transformations=['c_crop,w_100'])
        self.assertEqual(get_method(mocker), 'DELETE')
        self.assertEqual(get_param(mocker, 'transformations'), 'c_crop,w_100')
Ejemplo n.º 7
0
 def setUp(self):
     if TestApi.initialized: return
     TestApi.initialized = True
     cloudinary.reset_config()
     if not cloudinary.config().api_secret: return
     try:
         api.delete_resources(["api_test", "api_test2", "api_test3"])
     except:
         pass
     try:
         api.delete_transformation("api_test_transformation")
     except:
         pass
     uploader.upload("tests/logo.png",
                     public_id="api_test",
                     tags="api_test_tag",
                     eager=[{
                         "width": 100,
                         "crop": "scale"
                     }])
     uploader.upload("tests/logo.png",
                     public_id="api_test2",
                     tags="api_test_tag",
                     eager=[{
                         "width": 100,
                         "crop": "scale"
                     }])
Ejemplo n.º 8
0
 def tearDownClass(cls):
     try:
         api.delete_resources([
             API_TEST_ID, API_TEST_ID2, API_TEST_ID3, API_TEST_ID4,
             API_TEST_ID5
         ])
     except Exception:
         pass
     for transformation in [
             API_TEST_TRANS, API_TEST_TRANS2, API_TEST_TRANS3
     ]:
         try:
             api.delete_transformation(transformation)
         except Exception:
             pass
     for preset in [
             API_TEST_PRESET, API_TEST_PRESET2, API_TEST_PRESET3,
             API_TEST_PRESET4
     ]:
         try:
             api.delete_upload_preset(preset)
         except Exception:
             pass
     cloudinary.api.delete_resources_by_tag(UNIQUE_TAG)
     cloudinary.api.delete_resources_by_tag(UNIQUE_TAG, resource_type='raw')
Ejemplo n.º 9
0
def upload_files():
    print("--- Upload a local file")
    isDifferent = False
    ## foto que se tomo recientemente
    response = upload("img/escala.jpg", tags="")
    dump_response(response)
    ## se obtiene la imagen
    url, options = cloudinary_url(
        response['public_id'],
        format=response['format'],
    )
    public_id = response['public_id']
    ## abrir la imagen
    url_response = urllib.urlopen(url)
    img_array = np.array(bytearray(url_response.read()), dtype=np.uint8)
    duplicate = cv2.imdecode(img_array, -1)
    ## url de la imagen original 
    originalUrl = "https://res.cloudinary.com/choskas/image/upload/v1607385168/cg9lnefzs5wbdslubtzz.jpg"
    url_response2 = urllib.urlopen(originalUrl)
    img_array2 = np.array(bytearray(url_response2.read()), dtype=np.uint8)
    original = cv2.imdecode(img_array2, -1)
    print(duplicate, original)
    # duplicate = cv2.imread(img)
    image1 = original.shape
    image2 = duplicate.shape
    print(image1, image2)
    if original.shape == duplicate.shape:
        difference = cv2.subtract(original, duplicate)
        b, g, r = cv2.split(difference)
        print(b, g, r)
        if cv2.countNonZero(b) == 0 and cv2.countNonZero(g) == 0 and cv2.countNonZero(r) == 0:
            print("Las imagenes son iguales")
            delete_resources(public_id)
            url = ""
        else:
            ## correo en html
            html = f"""\
                    <html>
                    <head></head>
                    <body>
                    <p>Hola!<br>
                    Se ha detectado movimiento inusual<br>
                    <img width=300 height=200 src={url}> </img>
                    </p>
                    </body>
                    </html> 
                    """
            ## funciones para mandar emails        
            part2 = MIMEText(html, 'html')
            msg.attach(part2)
            s = smtplib.SMTP('smtp.gmail.com', 587)
            s.starttls()
            s.login('*****@*****.**', EMAIL_PASSWORD)
            s.sendmail(me, you, msg.as_string())
            s.quit()
            print("las imagenes son diferentes")
            print("mostando la imagen diferente ->")
            isDifferent = True
    return jsonify({"isComplete": True, "url": url, "isDifferent" : isDifferent})
Ejemplo n.º 10
0
 def test09_delete_resources(self, mocker):
     """ should allow deleting resources """
     mocker.return_value = MOCK_RESPONSE
     api.delete_resources([API_TEST_ID, API_TEST_ID2])
     args, kargs = mocker.call_args
     self.assertEqual(args[0], 'DELETE')
     self.assertTrue(get_uri(args).endswith('/resources/image/upload'))
     self.assertIn(('public_ids[]', API_TEST_ID), get_params(args))
     self.assertIn(('public_ids[]', API_TEST_ID2), get_params(args))
Ejemplo n.º 11
0
 def test09_delete_resources_tuple(self, mocker):
     """ should allow deleting resources """
     mocker.return_value = MOCK_RESPONSE
     api.delete_resources((API_TEST_ID, API_TEST_ID2,))
     args, kargs = mocker.call_args
     self.assertEqual(args[0], 'DELETE')
     self.assertTrue(get_uri(args).endswith('/resources/image/upload'))
     param = get_list_param(mocker, 'public_ids')
     self.assertIn(API_TEST_ID, param)
     self.assertIn(API_TEST_ID2, param)
Ejemplo n.º 12
0
    def process(self, image, results):
        result = upload(image)

        if result.get('url'):
            search = Client()
            search.add_image_urls([result.get('url')])
            search.make_request_data()
            search.make_request()
            search_results = search.get_results()

            results[result.get('url')] = search_results

            delete_resources(public_ids=[result.get('public_id')])
Ejemplo n.º 13
0
 def setUp(self):
   if TestApi.initialized: return
   TestApi.initialized = True
   cloudinary.reset_config()
   if not cloudinary.config().api_secret: return 
   try:
     api.delete_resources(["api_test", "api_test2", "api_test3"])
   except:
     pass
   try:
     api.delete_transformation("api_test_transformation")
   except:
     pass
   uploader.upload("tests/logo.png", public_id="api_test", tags="api_test_tag", eager=[{"width": 100,"crop": "scale"}])
   uploader.upload("tests/logo.png", public_id="api_test2", tags="api_test_tag", eager=[{"width": 100,"crop": "scale"}])
Ejemplo n.º 14
0
 def test_restore(self):
     """ should support restoring resources """
     uploader.upload("tests/logo.png", public_id="api_test_restore", backup=True)
     resource = api.resource("api_test_restore")
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 3381)
     api.delete_resources(["api_test_restore"])
     resource = api.resource("api_test_restore")
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 0)
     self.assertIs(resource["placeholder"], True)
     response = api.restore(["api_test_restore"])
     info = response["api_test_restore"]
     self.assertNotEqual(info, None)
     self.assertEqual(info["bytes"], 3381)
     resource = api.resource("api_test_restore")
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 3381)
Ejemplo n.º 15
0
 def test_restore(self):
     """ should support restoring resources """
     uploader.upload("tests/logo.png", public_id="api_test_restore", backup=True)
     resource = api.resource("api_test_restore")
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 3381)
     api.delete_resources(["api_test_restore"])
     resource = api.resource("api_test_restore")
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 0)
     self.assertIs(resource["placeholder"], True)
     response = api.restore(["api_test_restore"])
     info = response["api_test_restore"]
     self.assertNotEqual(info, None)
     self.assertEqual(info["bytes"], 3381)
     resource = api.resource("api_test_restore")
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 3381)
Ejemplo n.º 16
0
 def test_restore(self):
     """ should support restoring resources """
     uploader.upload(TEST_IMAGE, public_id=RESTORE_TEST_ID, backup=True, tags=[UNIQUE_API_TAG])
     resource = api.resource(RESTORE_TEST_ID)
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 3381)
     api.delete_resources([RESTORE_TEST_ID])
     resource = api.resource(RESTORE_TEST_ID)
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 0)
     self.assertIs(resource["placeholder"], True)
     response = api.restore([RESTORE_TEST_ID])
     info = response[RESTORE_TEST_ID]
     self.assertNotEqual(info, None)
     self.assertEqual(info["bytes"], 3381)
     resource = api.resource(RESTORE_TEST_ID)
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 3381)
Ejemplo n.º 17
0
 def setUp(self):
     if ApiTest.initialized: return
     ApiTest.initialized = True
     cloudinary.reset_config()
     if not cloudinary.config().api_secret: return
     try:
         api.delete_resources(["api_test", "api_test2", "api_test3"])
     except:
         pass
     for transformation in ["api_test_transformation", "api_test_transformation2", "api_test_transformation3",
                            "api_test_upload_preset", "api_test_upload_preset2", "api_test_upload_preset3",
                            "api_test_upload_preset4"]:          
       try:
           api.delete_transformation(transformation)
       except:
           pass
     uploader.upload("tests/logo.png", public_id="api_test", tags="api_test_tag", context="key=value", eager=[{"width": 100,"crop": "scale"}])
     uploader.upload("tests/logo.png", public_id="api_test2", tags="api_test_tag", context="key=value", eager=[{"width": 100,"crop": "scale"}])
Ejemplo n.º 18
0
    def test09c_delete_resources_by_transformations(self, mocker):
        """ should allow deleting resources by transformations """
        mocker.return_value = MOCK_RESPONSE

        api.delete_resources(['api_test', 'api_test2'], transformations=['c_crop,w_100'])
        self.assertEqual(get_method(mocker), 'DELETE')
        self.assertEqual(get_param(mocker, 'transformations'), 'c_crop,w_100')

        api.delete_all_resources(transformations=['c_crop,w_100', {"crop": "scale", "width": 107}])
        self.assertEqual(get_method(mocker), 'DELETE')
        self.assertEqual(get_param(mocker, 'transformations'), 'c_crop,w_100|c_scale,w_107')

        api.delete_resources_by_prefix("api_test_by", transformations='c_crop,w_100')
        self.assertEqual(get_method(mocker), 'DELETE')
        self.assertEqual(get_param(mocker, 'transformations'), 'c_crop,w_100')

        api.delete_resources_by_tag("api_test_tag", transformations=['c_crop,w_100'])
        self.assertEqual(get_method(mocker), 'DELETE')
        self.assertEqual(get_param(mocker, 'transformations'), 'c_crop,w_100')
Ejemplo n.º 19
0
    def _delete_unique_remote_files(self):
        if not len(self.unique_remote_file_names):
            return True

        if not (self.force or confirm_action(
                f"Running this command will delete {len(self.unique_remote_file_names)} remote files. "
                f"Continue? (y/N)")):
            return False

        logger.info(f"Deleting {len(self.unique_remote_file_names)} resources "
                    f"from Cloudinary folder '{self.remote_dir}'")
        files_to_delete_from_cloudinary = list(
            map(lambda x: self.remote_files[x], self.unique_remote_file_names))

        for i in product({"upload", "private", "authenticated"},
                         {"image", "video", "raw"}):
            batch = list(
                map(
                    lambda x: x['public_id'],
                    filter(
                        lambda x: x["type"] == i[0] and x["resource_type"] ==
                        i[1], files_to_delete_from_cloudinary)))
            if not len(batch):
                continue

            logger.info(
                "Deleting {} resources with type '{}' and resource_type '{}'".
                format(len(batch), *i))
            counter = 0
            while counter * DELETE_ASSETS_BATCH_SIZE < len(batch) and len(
                    batch) > 0:
                counter += 1
                res = api.delete_resources(
                    batch[(counter - 1) * DELETE_ASSETS_BATCH_SIZE:counter *
                          DELETE_ASSETS_BATCH_SIZE],
                    invalidate=True,
                    resource_type=i[1],
                    type=i[0])
                num_deleted = reduce(
                    lambda x, y: x + 1
                    if y == "deleted" else x, res['deleted'].values(), 0)
                if self.verbose:
                    print_json(res)
                if num_deleted != len(batch):
                    logger.error("Failed deletes:\n{}".format("\n".join(
                        list(
                            map(
                                lambda x: x[0],
                                filter(lambda x: x[1] != 'deleted',
                                       res['deleted'].items()))))))
                else:
                    logger.info(
                        style(f"Deleted {num_deleted} resources", fg="green"))

        return True
Ejemplo n.º 20
0
 def test_restore(self):
     """ should support restoring resources """
     TEST_ID = "api_test_restore_{}".format(SUFFIX)
     uploader.upload("tests/logo.png",
                     public_id=TEST_ID,
                     backup=True,
                     tags=[UNIQUE_TAG])
     resource = api.resource(TEST_ID)
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 3381)
     api.delete_resources([TEST_ID])
     resource = api.resource(TEST_ID)
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 0)
     self.assertIs(resource["placeholder"], True)
     response = api.restore([TEST_ID])
     info = response[TEST_ID]
     self.assertNotEqual(info, None)
     self.assertEqual(info["bytes"], 3381)
     resource = api.resource(TEST_ID)
     self.assertNotEqual(resource, None)
     self.assertEqual(resource["bytes"], 3381)
Ejemplo n.º 21
0
    def tearDownClass(cls):
        try:
            api.delete_resources([
                API_TEST_ID, API_TEST_ID2, API_TEST_ID3, API_TEST_ID4,
                API_TEST_ID5
            ])
        except Exception:
            pass
        for transformation in [
                API_TEST_TRANS, API_TEST_TRANS2, API_TEST_TRANS3,
                API_TEST_TRANS_SCALE100_STR
        ]:
            try:
                api.delete_transformation(transformation)
            except Exception:
                pass
        cloudinary.api.delete_resources_by_tag(UNIQUE_API_TAG)
        cloudinary.api.delete_resources_by_tag(UNIQUE_API_TAG,
                                               resource_type='raw')

        try:
            api.delete_upload_mapping(MAPPING_TEST_ID)
        except Exception:
            pass
Ejemplo n.º 22
0
    def tearDownClass(cls):
        try:
            api.delete_resources([API_TEST_ID, API_TEST_ID2, API_TEST_ID3, API_TEST_ID4, API_TEST_ID5])
        except Exception:
            pass
        for transformation in [API_TEST_TRANS, API_TEST_TRANS2, API_TEST_TRANS3]:
            try:
                api.delete_transformation(transformation)
            except Exception:
                pass
        presets_response = api.upload_presets(max_results=200)
        preset_names = [ preset["name"] for preset in presets_response.get('presets',[]) if UNIQUE_TAG in preset.get('settings',{}).get('tags','')]
        for name in preset_names:
            try:
                api.delete_upload_preset(name)
            except Exception:
                pass
        cloudinary.api.delete_resources_by_tag(UNIQUE_TAG)
        cloudinary.api.delete_resources_by_tag(UNIQUE_TAG, resource_type='raw')

        try:
            api.delete_upload_mapping(MAPPING_TEST_ID)
        except Exception:
            pass
Ejemplo n.º 23
0
 def delete(self, public_ids):
     # TODO: error handling
     api.delete_resources(public_ids)
Ejemplo n.º 24
0
 def tearDown(self) -> None:
     api.delete_resources([f'event_pics/{self.e1.title}'])
Ejemplo n.º 25
0
def photo_pre_delete(sender, instance, using, **kwargs):
    api.delete_resources([instance.image.public_id])

    key = "photo:%s" % instance.artwork.slug
    cache.delete(key)
Ejemplo n.º 26
0
 def tearDown(self):
     api.delete_resources([API_TEST_ID])
Ejemplo n.º 27
0
def cleanup_test_resources(params):
    for public_ids_with_options in params:
        options = public_ids_with_options[1] if len(public_ids_with_options) > 1 else {}
        with ignore_exception():
            api.delete_resources(public_ids_with_options[0], **options)
Ejemplo n.º 28
0
def photo_pre_delete(sender, instance, using, **kwargs):
    api.delete_resources([instance.image.public_id])

    key = "photo:%s" % instance.artwork.slug
    cache.delete(key)
Ejemplo n.º 29
0
def read_delete(public_id: str = Form(...)):
    res = delete_resources(public_ids=[public_id])
    return res
Ejemplo n.º 30
0
def sync(local_folder, cloudinary_folder, push, pull, verbose, expression):
    if push == pull:
        print("Please use either the '--push' OR '--pull' options")
        exit(1)

    etag = lambda f: md5(open(f, 'rb').read()).hexdigest()

    def walk_dir(folder):
        all_files = {}
        for root, _, files in walk(folder):
            for _file in files:
                all_files[splitext(path_join(root, _file)[len(folder) + 1:])[0]] = {
                    "etag": etag(path_join(root, _file)), "path": path_join(root, _file)}
        return all_files

    def query_cld_folder(folder):
        next_cursor = None
        items = {}
        while True:
            search_expr = "{}/*".format(folder)
            if expression:
                search_expr = "{0} AND {1}".format(search_expr, expression)
            res = Search().expression(search_expr).next_cursor(next_cursor).with_field(
                "image_analysis").max_results(500).execute()
            for item in res['resources']:
                items[item['public_id'][len(folder) + 1:]] = {"etag": item['image_analysis']['etag'],
                                                              "resource_type": item['resource_type'],
                                                              "public_id": item['public_id'], "type": item['type'],
                                                              "format": item['format']}
            if 'next_cursor' not in res.keys():
                break
            else:
                next_cursor = res['next_cursor']
        return items

    files = walk_dir(abspath(local_folder))
    print("Found {} items in local folder '{}'".format(len(files.keys()), local_folder))
    cld_files = query_cld_folder(cloudinary_folder)
    print("Found {} items in Cloudinary folder '{}'".format(len(cld_files.keys()), cloudinary_folder))
    files_ = set(files.keys())
    cld_files_ = set(cld_files.keys())

    files_in_cloudinary_nin_local = cld_files_ - files_
    files_in_local_nin_cloudinary = files_ - cld_files_
    skipping = 0

    if push:

        files_to_delete_from_cloudinary = list(cld_files_ - files_)
        files_to_push = files_ - cld_files_
        files_to_check = files_ - files_to_push
        print("\nCalculating differences...\n")
        for f in files_to_check:
            if files[f]['etag'] == cld_files[f]['etag']:
                if verbose:
                    print(F_WARN("{} already exists in Cloudinary".format(f)))
                skipping += 1
            else:
                files_to_push.add(f)
        print("Skipping upload for {} items".format(skipping))
        if len(files_to_delete_from_cloudinary) > 0:
            print("Deleting {} resources from Cloudinary folder '{}'".format(len(files_to_delete_from_cloudinary),
                                                                             cloudinary_folder))
            files_to_delete_from_cloudinary = list(map(lambda x: cld_files[x], files_to_delete_from_cloudinary))

            for i in product({"upload", "private", "authenticated"}, {"image", "video", "raw"}):
                batch = list(map(lambda x: x['public_id'],
                                 filter(lambda x: x["type"] == i[0] and x["resource_type"] == i[1],
                                        files_to_delete_from_cloudinary)))
                if len(batch) > 0:
                    print("Deleting {} resources with type '{}' and resource_type '{}'".format(len(batch), *i))
                    counter = 0
                    while counter * 100 < len(batch) and len(batch) > 0:
                        counter += 1
                        res = api.delete_resources(batch[(counter - 1) * 100:counter * 100], invalidate=True,
                                                   resource_type=i[1], type=i[0])
                        num_deleted = reduce(lambda x, y: x + 1 if y == "deleted" else x, res['deleted'].values(), 0)
                        if verbose:
                            log(res)
                        if num_deleted != len(batch):
                            print(F_FAIL("Failed deletes:\n{}".format("\n".join(list(
                                map(lambda x: x[0], filter(lambda x: x[1] != 'deleted', res['deleted'].items())))))))
                        else:
                            print(F_OK("Deleted {} resources".format(num_deleted)))

        to_upload = list(filter(lambda x: split(x)[1][0] != ".", files_to_push))
        print("Uploading {} items to Cloudinary folder '{}'".format(len(to_upload), cloudinary_folder))

        threads = []

        def threaded_upload(options, path, verbose):
            res = _uploader.upload(path, **options)
            if verbose:
                print(F_OK("Uploaded '{}'".format(res['public_id'])))

        for i in to_upload:
            modif_folder = path_join(cloudinary_folder, sep.join(i.split(sep)[:-1]))
            options = {'use_filename': True, 'unique_filename': False, 'folder': modif_folder, 'invalidate': True,
                       'resource_type': 'auto'}
            threads.append(Thread(target=threaded_upload, args=(options, files[i]['path'], verbose)))

        for t in threads:
            while active_count() >= 30:
                # prevent concurrency overload
                sleep(1)
            t.start()
            sleep(1 / 10)

        [t.join() for t in threads]

        print("Done!")

    else:
        files_to_delete_local = list(files_in_local_nin_cloudinary)
        files_to_pull = files_in_cloudinary_nin_local
        files_to_check = cld_files_ - files_to_pull

        print("\nCalculating differences...\n")
        for f in files_to_check:
            if files[f]['etag'] == cld_files[f]['etag']:
                if verbose:
                    print(F_WARN("{} already exists locally".format(f)))
                skipping += 1
            else:
                files_to_pull.add(f)
        print("Skipping download for {} items".format(skipping))

        def delete_empty_folders(root, verbose, remove_root=False):
            if not isdir(root):
                return

            files = listdir(root)
            if len(files):
                for f in files:
                    fullpath = path_join(root, f)
                    if isdir(fullpath):
                        delete_empty_folders(fullpath, verbose, True)

            files = listdir(root)
            if len(files) == 0 and remove_root:
                if verbose:
                    print("Removing empty folder '{}'".format(root))
                rmdir(root)

        def create_required_directories(root, verbose):
            if isdir(root):
                return
            else:
                create_required_directories(sep.join(root.split(sep)[:-1]), verbose)
                if verbose:
                    print("Creating directory '{}'".format(root))
                mkdir(root)

        print("Deleting {} local files...".format(len(files_to_delete_local)))
        for i in files_to_delete_local:
            remove(abspath(files[i]['path']))
            if verbose:
                print("Deleted '{}'".format(abspath(files[i]['path'])))

        print("Deleting empty folders...")

        delete_empty_folders(local_folder, verbose)

        print("Downloading {} files from Cloudinary".format(len(files_to_pull)))

        threads = []

        def threaded_pull(local_path, verbose, cld_files):
            with open(local_path, "wb") as f:
                to_download = cld_files[i]
                r = get(cld_url(to_download['public_id'], resource_type=to_download['resource_type'],
                                type=to_download['type'])[0])
                f.write(r.content)
                f.close()
            if verbose:
                print(F_OK("Downloaded '{}' to '{}'".format(i, local_path)))

        for i in files_to_pull:
            local_path = abspath(path_join(local_folder,
                                           i + "." + cld_files[i]['format']
                                           if cld_files[i]['resource_type'] != 'raw' else i))
            create_required_directories(split(local_path)[0], verbose)

            threads.append(Thread(target=threaded_pull, args=(local_path, verbose, cld_files)))

        for t in threads:
            while active_count() >= 30:
                # prevent concurrency overload
                sleep(1)
            t.start()
            sleep(1 / 10)

        [t.join() for t in threads]

        print("Done!")
Ejemplo n.º 31
0
def cleanup_test_resources(params):
    for public_ids_with_options in params:
        options = public_ids_with_options[1] if len(public_ids_with_options) > 1 else {}
        with ignore_exception():
            api.delete_resources(public_ids_with_options[0], **options)