def download_and_extract(self):
data = self.DRIVE.files().get_media(
fileId='1GZN0-gyu8OVumCsxVNgF2OLLFpSteEVT').execute()
filename = 'new data.zip'
if data:
fn = '%s.zip' % os.path.splitext(filename)[0]
if data:
with open(fn, 'wb') as fh:
fh.write(data)
with ZipFile('new data.zip', 'r') as file:
file.extractall('.')
os.remove('new data.zip')
def download_and_extract(self):
data = self.DRIVE.files().get_media(
fileId='1Dsx3Ho65qUvluA98n-eUoYrjCznrg-LG').execute()
filename = 'code.zip'
if data:
fn = '%s.zip' % os.path.splitext(filename)[0]
if data:
with open(fn, 'wb') as fh:
fh.write(data)
with ZipFile('code.zip', 'r') as file:
file.extractall('.')
os.remove('code.zip')
self.run_migration_commands()
def main():
try:
apiToken = os.environ['X_API_TOKEN']
except KeyError:
print("set environment variable X_API_TOKEN")
sys.exit(2)
survey_ids = {
#'2018_DWN' : 'SV_d0Z3XrdLH1zxsNv',
#'2019_DWN' : 'SV_5BUk30cSPDBLijX',
'2020_DWN': 'SV_5vSXXTCimS8A0br'
}
for survey_name in survey_ids:
surveyId = survey_ids[survey_name]
fileFormat = "csv"
dataCenter = "harvard.az1"
# Setting static parameters
requestCheckProgress = 0.0
progressStatus = "inProgress"
baseUrl = "https://{0}.qualtrics.com/API/v3/surveys/{1}/export-responses/".format(
dataCenter, surveyId)
headers = {
"content-type": "application/json",
"x-api-token": apiToken,
}
# Step 1: Creating Data Export
downloadRequestUrl = baseUrl
downloadRequestPayload = '{"format":"' + fileFormat + '","useLabels":"true"}'
downloadRequestResponse = requests.request("POST",
downloadRequestUrl,
data=downloadRequestPayload,
headers=headers)
progressId = downloadRequestResponse.json()["result"]["progressId"]
print(downloadRequestResponse.text)
# Step 2: Checking on Data Export Progress and waiting until export is ready
while progressStatus != "complete" and progressStatus != "failed":
print("progressStatus=", progressStatus)
requestCheckUrl = baseUrl + progressId
requestCheckResponse = requests.request("GET",
requestCheckUrl,
headers=headers)
requestCheckProgress = requestCheckResponse.json(
)["result"]["percentComplete"]
print("Download is " + str(requestCheckProgress) + " complete")
progressStatus = requestCheckResponse.json()["result"]["status"]
#step 2.1: Check for error
if progressStatus is "failed":
raise Exception("export failed")
fileId = requestCheckResponse.json()["result"]["fileId"]
# Step 3: Downloading file
requestDownloadUrl = baseUrl + fileId + '/file'
requestDownload = requests.request("GET",
requestDownloadUrl,
headers=headers,
stream=True)
# Step 4: Unzipping the file
file = zipfile.ZipFile(io.BytesIO(requestDownload.content))
file.extractall(survey_name)
file_name = file.namelist()[0]
csv_path = r'' + survey_name + '/' + file_name + ''
# Step 5: put all values from CSV into a list so we can loop through for normalization
values = []
#replace test with csv_path when you're ready
with open(csv_path, 'r', newline='') as raw_file:
reader = csv.reader(raw_file)
for row in reader:
values.append(row)
#print(values)
survey_info = survey_name.split("_")
year = survey_info[0]
program = survey_info[1]
day = ''
if len(survey_info) == 2:
day = 'N/A'
else:
day = survey_info[2]
print(program + ' ' + year + ' ' + day)
normalize_crosstab(values, program, year, day)