def handler(self, collection, obj, set_field, base_dir, task_args=None, grab_args=None, callback=None): for image in obj.get(set_field, []): path = hashed_path(image["url"], base_dir=base_dir) if os.path.exists(path): if path != image["path"]: db[collection].update( {"_id": obj["_id"], ("%s.url" % set_field): image["url"]}, {"$set": {"%s.$.path": path}} ) else: kwargs = {} if task_args: kwargs = deepcopy(task_args) g = Grab() g.setup(url=image["url"]) if grab_args: g.setup(**grab_args) g.setup(referer=build_image_hosting_referer(image["url"])) yield Task( callback=callback or image_set_handler, grab=g, collection=collection, path=path, obj=obj, image=image, set_field=set_field, disable_cache=True, backup=g.dump_config(), **kwargs )
def handler(self, url, collection, obj, path_field, base_dir, task_args=None, grab_args=None, callback=None): path = hashed_path(url, base_dir=base_dir) if os.path.exists(path): if path != obj.get(path_field, None): db[collection].update({"_id": obj["_id"]}, {"$set": {path_field: path}}) else: kwargs = {} if task_args: kwargs = deepcopy(task_args) g = Grab() g.setup(url=url) if grab_args: g.setup(**grab_args) g.setup(referer=build_image_hosting_referer(url)) yield Task( callback=callback or image_handler, grab=g, collection=collection, path=path, obj=obj, path_field=path_field, disable_cache=True, backup=g.dump_config(), **kwargs )
def handler(self, collection, obj, set_field, base_dir, task_args=None, grab_args=None, callback=None): from database import db for image in obj.get(set_field, []): path = hashed_path(image['url'], base_dir=base_dir) if os.path.exists(path): if path != image['path']: db[collection].update( {'_id': obj['_id'], ('%s.url' % set_field): image['url']}, {'$set': {('%s.$.path' % set_field): path}}) else: kwargs = {} if task_args: kwargs = deepcopy(task_args) g = Grab() g.setup(url=image['url']) if grab_args: g.setup(**grab_args) g.setup(referer=build_image_hosting_referer(image['url'])) yield Task( callback=callback or image_set_handler, grab=g, collection=collection, path=path, obj=obj, image=image, set_field=set_field, disable_cache=True, backup=g.dump_config(), **kwargs )
def handler(self, collection, obj, set_field, base_dir, task_args=None, grab_args=None, callback=None): from database import db for image in obj.get(set_field, []): path = hashed_path(image['url'], base_dir=base_dir) if os.path.exists(path): if path != image['path']: db[collection].update( { '_id': obj['_id'], ('%s.url' % set_field): image['url'] }, {'$set': { '%s.$.path': path }}) else: kwargs = {} if task_args: kwargs = deepcopy(task_args) g = Grab() g.setup(url=image['url']) if grab_args: g.setup(**grab_args) g.setup(referer=build_image_hosting_referer(image['url'])) yield Task(callback=callback or image_set_handler, grab=g, collection=collection, path=path, obj=obj, image=image, set_field=set_field, disable_cache=True, backup=g.dump_config(), **kwargs)
def wrapper(*args, **kwargs): grab = Grab(debug_post=True, log_dir='../log') ## load grab instance config = {} # config = {} # config = User.grab_config try: grab.load_config(config) except Exception: logging.debug("Configuration is empty!") # login() ## check login email = '' # User.email password = '' #User.password if not grab.cookies.cookiejar: if not login(grab, email, password): logging.error("ERROR! Not authorised") return logging.debug("Successfully logged in") config = grab.dump_config() logging.debug("Session restored successfully") return func(grab=grab, *args, **kwargs)
def handler(self, url, collection, obj, path_field, base_dir, task_args=None, grab_args=None, callback=None): from database import db path = hashed_path(url, base_dir=base_dir) if os.path.exists(path): if path != obj.get(path_field, None): db[collection].update({'_id': obj['_id']}, {'$set': { path_field: path }}) else: kwargs = {} if task_args: kwargs = deepcopy(task_args) g = Grab() g.setup(url=url) if grab_args: g.setup(**grab_args) g.setup(referer=build_image_hosting_referer(url)) yield Task(callback=callback or image_handler, grab=g, collection=collection, path=path, obj=obj, path_field=path_field, disable_cache=True, backup=g.dump_config(), **kwargs)