# set of utilities to interact with files
# @author: rm3086 (at) columbia (dot) edu
import csv, shutil, os, sys, glob, pickle, string
csv.field_size_limit(2147483647)
from log import strd_logger
# logger
global log
log = strd_logger ('file')
# check if a file exist
def file_exist (fname):
try:
open(fname,'r')
return True
except IOError:
return False
# create directory if not existing
def mkdir (dirname):
try:
os.makedirs(dirname)
except OSError:
pass
except Exception as e:
log.error (e)
return False
return True
# set of general utilities to interact with the Web
# @author: rm3086 (at) columbia (dot) edu
import urllib2, urllib3, json
from log import strd_logger
# logger
log = strd_logger('web')
# get the html source associated to a URL
def download_web_data(url):
try:
con = urllib3.connection_from_url(url)
html = con.urlopen('GET', url, retries=500, timeout=10)
con.close()
return html.data
except Exception as e:
log.error('%s: %s' % (e, url))
return None
# get json data
def download_json_data(url):
try:
con = urllib2.urlopen(url)
data = con.read()
con.close()
return json.loads(data)
except Exception as e:
log.error('%s: %s' % (e, url))
#basic operations for files
#original author: rm3086 (at) columbia (dot) edu
#added and modified by Handong Ma: hm2588 (at) columbia (dot) edu
import csv, shutil, os, glob, cPickle
from log import strd_logger
import re
import sys
import inspect
# logger
global log
log = strd_logger('file')
#$$$$$$$$$#
def ifContinue():
decision = raw_input('Wait for your command: continue(c) or stop(any key)')
if decision == 'c':
return True
else:
sys.exit("stopped")
#$$$$$$$# check if a file exist
def file_exist (fname):
try:
open(fname,'r')
return True
except IOError:
return False
# set of general utilities to interact with the Web
# @author: rm3086 (at) columbia (dot) edu
import urllib2, urllib3, json
from log import strd_logger
# logger
log = strd_logger('web')
# get the html source associated to a URL
def download_web_data(url):
try:
con = urllib3.connection_from_url(url)
html = con.urlopen('GET', url, retries=500, timeout=10)
con.close()
return html.data
except Exception as e:
log.error('%s: %s' % (e, url))
return None
# get json data
def download_json_data(url):
try:
con = urllib2.urlopen(url)
data = con.read()
con.close()
return json.loads(data)
except Exception as e:
# basic operations for files
# original author: rm3086 (at) columbia (dot) edu
# added and modified by Handong Ma: hm2588 (at) columbia (dot) edu
import csv, shutil, os, glob, cPickle
from log import strd_logger
import re
# logger
global log
log = strd_logger("file")
# check the dir name whether is standard with a '/' in the end
def check_dir(dirName):
if re.findall("/$", dirName):
return str(dirName)
else:
return str(dirName + "/")
# check if a file exist
def file_exist(fname):
try:
open(fname, "r")
return True
except IOError:
return False
# create directory if not existing
def mkdir(dirname):
