def download_an_image(url, file_path, exclude_url=PARSER_EXCLUDE_URL):
"""
:param url: image's url
:param file_path: path to save image
:return: {"200", "Zero", "Error", "link-anh-ko-hop-le", "Small", exception_message}
If url is in 'exclude_url' list : Stop downloading and return
Get html content and write to file
Check downloaded file's status and return it
"""
if re.search("|".join(exclude_url), url) is not None:
print "{0} : link anh ko hop le".format(url)
return "link-anh-ko-hop-le"
if SHOW_DOWNLOAD_STATUS:
print "[{0}] {1} : Downloading".format(utils.get_current_time(), url)
html_content = utils.urllib2_get(url=url)
utils.write_string_to_file(file_path, html_content.read())
if utils.get_file_size(file_path) == int(html_content.info()['Content-Length']):
if utils.get_image_resolution(file_path) < 300:
return "Small"
else:
return "200"
elif utils.get_file_size(file_path) == 0:
return "Zero"
else:
return "Error"
def mongo_backup_run(collection):
x = _mongo_credential
dump_dir = "{0}/{1}".format(MONGO_BACKUP_PATH, utils.get_current_time())
print dump_dir
cmd = 'mongodump --host {0} --port {1} -d {2} -u {3} -p {4} -c {5} -o {6} --authenticationDatabase admin'.format(
x['DB_HOST'], x['DB_PORT'], x['DB_NAME'], x['DB_USER'], x['DB_PASS'], collection, dump_dir)
call(cmd.split())
def get_url_status(url):
"""
Get html status code
:param url:
:return:
"""
status = utils.urllib2_get(url=url).getcode()
if SHOW_DOWNLOAD_STATUS:
print "[{0}] {1} : {2}".format(utils.get_current_time(), url, status)
return str(status)
