def handler(self, collection, obj, set_field, base_dir, task_args=None, grab_args=None, callback=None):
for image in obj.get(set_field, []):
path = hashed_path(image["url"], base_dir=base_dir)
if os.path.exists(path):
if path != image["path"]:
db[collection].update(
{"_id": obj["_id"], ("%s.url" % set_field): image["url"]}, {"$set": {"%s.$.path": path}}
)
else:
kwargs = {}
if task_args:
kwargs = deepcopy(task_args)
g = Grab()
g.setup(url=image["url"])
if grab_args:
g.setup(**grab_args)
g.setup(referer=build_image_hosting_referer(image["url"]))
yield Task(
callback=callback or image_set_handler,
grab=g,
collection=collection,
path=path,
obj=obj,
image=image,
set_field=set_field,
disable_cache=True,
backup=g.dump_config(),
**kwargs
)
def handler(self, url, collection, obj, path_field, base_dir, task_args=None, grab_args=None, callback=None):
path = hashed_path(url, base_dir=base_dir)
if os.path.exists(path):
if path != obj.get(path_field, None):
db[collection].update({"_id": obj["_id"]}, {"$set": {path_field: path}})
else:
kwargs = {}
if task_args:
kwargs = deepcopy(task_args)
g = Grab()
g.setup(url=url)
if grab_args:
g.setup(**grab_args)
g.setup(referer=build_image_hosting_referer(url))
yield Task(
callback=callback or image_handler,
grab=g,
collection=collection,
path=path,
obj=obj,
path_field=path_field,
disable_cache=True,
backup=g.dump_config(),
**kwargs
)
def handler(self, collection, obj, set_field, base_dir, task_args=None,
grab_args=None, callback=None):
from database import db
for image in obj.get(set_field, []):
path = hashed_path(image['url'], base_dir=base_dir)
if os.path.exists(path):
if path != image['path']:
db[collection].update(
{'_id': obj['_id'], ('%s.url' % set_field): image['url']},
{'$set': {('%s.$.path' % set_field): path}})
else:
kwargs = {}
if task_args:
kwargs = deepcopy(task_args)
g = Grab()
g.setup(url=image['url'])
if grab_args:
g.setup(**grab_args)
g.setup(referer=build_image_hosting_referer(image['url']))
yield Task(
callback=callback or image_set_handler,
grab=g,
collection=collection,
path=path,
obj=obj,
image=image,
set_field=set_field,
disable_cache=True,
backup=g.dump_config(),
**kwargs
)
def handler(self,
collection,
obj,
set_field,
base_dir,
task_args=None,
grab_args=None,
callback=None):
from database import db
for image in obj.get(set_field, []):
path = hashed_path(image['url'], base_dir=base_dir)
if os.path.exists(path):
if path != image['path']:
db[collection].update(
{
'_id': obj['_id'],
('%s.url' % set_field): image['url']
}, {'$set': {
'%s.$.path': path
}})
else:
kwargs = {}
if task_args:
kwargs = deepcopy(task_args)
g = Grab()
g.setup(url=image['url'])
if grab_args:
g.setup(**grab_args)
g.setup(referer=build_image_hosting_referer(image['url']))
yield Task(callback=callback or image_set_handler,
grab=g,
collection=collection,
path=path,
obj=obj,
image=image,
set_field=set_field,
disable_cache=True,
backup=g.dump_config(),
**kwargs)
def wrapper(*args, **kwargs):
grab = Grab(debug_post=True, log_dir='../log')
## load grab instance
config = {} # config = {} # config = User.grab_config
try:
grab.load_config(config)
except Exception:
logging.debug("Configuration is empty!")
# login()
## check login
email = '' # User.email
password = '' #User.password
if not grab.cookies.cookiejar:
if not login(grab, email, password):
logging.error("ERROR! Not authorised")
return
logging.debug("Successfully logged in")
config = grab.dump_config()
logging.debug("Session restored successfully")
return func(grab=grab, *args, **kwargs)
def handler(self,
url,
collection,
obj,
path_field,
base_dir,
task_args=None,
grab_args=None,
callback=None):
from database import db
path = hashed_path(url, base_dir=base_dir)
if os.path.exists(path):
if path != obj.get(path_field, None):
db[collection].update({'_id': obj['_id']},
{'$set': {
path_field: path
}})
else:
kwargs = {}
if task_args:
kwargs = deepcopy(task_args)
g = Grab()
g.setup(url=url)
if grab_args:
g.setup(**grab_args)
g.setup(referer=build_image_hosting_referer(url))
yield Task(callback=callback or image_handler,
grab=g,
collection=collection,
path=path,
obj=obj,
path_field=path_field,
disable_cache=True,
backup=g.dump_config(),
**kwargs)
